Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5313: move preservePositionIncrements from setter to ctor 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1537038 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2013-10-30 11:11:17 +00:00
parent ca248b5847
commit 9ba1122f55
7 changed files with 72 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -195,6 +195,9 @@ API Changes:
* LUCENE-5157: Rename OrdinalMap methods to clarify API and internal structure. * LUCENE-5157: Rename OrdinalMap methods to clarify API and internal structure.
(Boaz Leskes via Adrien Grand) (Boaz Leskes via Adrien Grand)
* LUCENE-5313: Move preservePositionIncrements from setter to ctor in
Analyzing/FuzzySuggester. (Areek Zillur via Mike McCandless)
Optimizations Optimizations
* LUCENE-5225: The ToParentBlockJoinQuery only keeps tracks of the the child * LUCENE-5225: The ToParentBlockJoinQuery only keeps tracks of the the child

32
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
View File

@ -74,8 +74,9 @@ import org.apache.lucene.util.fst.Util;
* then the partial text "ghost chr..." could see the * then the partial text "ghost chr..." could see the
* suggestion "The Ghost of Christmas Past". Note that * suggestion "The Ghost of Christmas Past". Note that
* position increments MUST NOT be preserved for this example * position increments MUST NOT be preserved for this example
* to work, so you should call * to work, so you should call the constructor with
* {@link #setPreservePositionIncrements(boolean) setPreservePositionIncrements(false)}. * <code>preservePositionIncrements</code> parameter set to
* false
* *
* <p> * <p>
* If SynonymFilter is used to map wifi and wireless network to * If SynonymFilter is used to map wifi and wireless network to
@ -145,14 +146,14 @@ public class AnalyzingSuggester extends Lookup {
private final boolean preserveSep; private final boolean preserveSep;
/** Include this flag in the options parameter to {@link /** Include this flag in the options parameter to {@link
* #AnalyzingSuggester(Analyzer,Analyzer,int,int,int)} to always * #AnalyzingSuggester(Analyzer,Analyzer,int,int,int,boolean)} to always
* return the exact match first, regardless of score. This * return the exact match first, regardless of score. This
* has no performance impact but could result in * has no performance impact but could result in
* low-quality suggestions. */ * low-quality suggestions. */
public static final int EXACT_FIRST = 1; public static final int EXACT_FIRST = 1;
/** Include this flag in the options parameter to {@link /** Include this flag in the options parameter to {@link
* #AnalyzingSuggester(Analyzer,Analyzer,int,int,int)} to preserve * #AnalyzingSuggester(Analyzer,Analyzer,int,int,int,boolean)} to preserve
* token separators when matching. */ * token separators when matching. */
public static final int PRESERVE_SEP = 2; public static final int PRESERVE_SEP = 2;
@ -187,21 +188,21 @@ public class AnalyzingSuggester extends Lookup {
private boolean preservePositionIncrements; private boolean preservePositionIncrements;
/** /**
* Calls {@link #AnalyzingSuggester(Analyzer,Analyzer,int,int,int) * Calls {@link #AnalyzingSuggester(Analyzer,Analyzer,int,int,int,boolean)
* AnalyzingSuggester(analyzer, analyzer, EXACT_FIRST | * AnalyzingSuggester(analyzer, analyzer, EXACT_FIRST |
* PRESERVE_SEP, 256, -1)} * PRESERVE_SEP, 256, -1, false)}
*/ */
public AnalyzingSuggester(Analyzer analyzer) { public AnalyzingSuggester(Analyzer analyzer) {
this(analyzer, analyzer, EXACT_FIRST | PRESERVE_SEP, 256, -1); this(analyzer, analyzer, EXACT_FIRST | PRESERVE_SEP, 256, -1, false);
} }
/** /**
* Calls {@link #AnalyzingSuggester(Analyzer,Analyzer,int,int,int) * Calls {@link #AnalyzingSuggester(Analyzer,Analyzer,int,int,int,boolean)
* AnalyzingSuggester(indexAnalyzer, queryAnalyzer, EXACT_FIRST | * AnalyzingSuggester(indexAnalyzer, queryAnalyzer, EXACT_FIRST |
* PRESERVE_SEP, 256, -1)} * PRESERVE_SEP, 256, -1, false)}
*/ */
public AnalyzingSuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer) { public AnalyzingSuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer) {
this(indexAnalyzer, queryAnalyzer, EXACT_FIRST | PRESERVE_SEP, 256, -1); this(indexAnalyzer, queryAnalyzer, EXACT_FIRST | PRESERVE_SEP, 256, -1, false);
} }
/** /**
@ -219,8 +220,11 @@ public class AnalyzingSuggester extends Lookup {
* @param maxGraphExpansions Maximum number of graph paths * @param maxGraphExpansions Maximum number of graph paths
* to expand from the analyzed form. Set this to -1 for * to expand from the analyzed form. Set this to -1 for
* no limit. * no limit.
* @param preservePositionIncrements Whether position holes
* should appear in the automata
*/ */
public AnalyzingSuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer, int options, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions) { public AnalyzingSuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer, int options, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions,
boolean preservePositionIncrements) {
this.indexAnalyzer = indexAnalyzer; this.indexAnalyzer = indexAnalyzer;
this.queryAnalyzer = queryAnalyzer; this.queryAnalyzer = queryAnalyzer;
if ((options & ~(EXACT_FIRST | PRESERVE_SEP)) != 0) { if ((options & ~(EXACT_FIRST | PRESERVE_SEP)) != 0) {
@ -242,12 +246,6 @@ public class AnalyzingSuggester extends Lookup {
throw new IllegalArgumentException("maxGraphExpansions must -1 (no limit) or > 0 (got: " + maxGraphExpansions + ")"); throw new IllegalArgumentException("maxGraphExpansions must -1 (no limit) or > 0 (got: " + maxGraphExpansions + ")");
} }
this.maxGraphExpansions = maxGraphExpansions; this.maxGraphExpansions = maxGraphExpansions;
preservePositionIncrements = true;
}
/** Whether to take position holes (position increment > 1) into account when
* building the automaton, <code>true</code> by default. */
public void setPreservePositionIncrements(boolean preservePositionIncrements) {
this.preservePositionIncrements = preservePositionIncrements; this.preservePositionIncrements = preservePositionIncrements;
} }

9
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FuzzySuggester.java
View File

@ -124,7 +124,7 @@ public final class FuzzySuggester extends AnalyzingSuggester {
* Analyzer that will be used for analyzing query text during lookup * Analyzer that will be used for analyzing query text during lookup
*/ */
public FuzzySuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer) { public FuzzySuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer) {
this(indexAnalyzer, queryAnalyzer, EXACT_FIRST | PRESERVE_SEP, 256, -1, DEFAULT_MAX_EDITS, DEFAULT_TRANSPOSITIONS, this(indexAnalyzer, queryAnalyzer, EXACT_FIRST | PRESERVE_SEP, 256, -1, false, DEFAULT_MAX_EDITS, DEFAULT_TRANSPOSITIONS,
DEFAULT_NON_FUZZY_PREFIX, DEFAULT_MIN_FUZZY_LENGTH, DEFAULT_UNICODE_AWARE); DEFAULT_NON_FUZZY_PREFIX, DEFAULT_MIN_FUZZY_LENGTH, DEFAULT_UNICODE_AWARE);
} }
@ -143,6 +143,7 @@ public final class FuzzySuggester extends AnalyzingSuggester {
* @param maxGraphExpansions Maximum number of graph paths * @param maxGraphExpansions Maximum number of graph paths
* to expand from the analyzed form. Set this to -1 for * to expand from the analyzed form. Set this to -1 for
* no limit. * no limit.
* @param preservePositionIncrements Whether position holes should appear in the automaton
* @param maxEdits must be >= 0 and <= {@link LevenshteinAutomata#MAXIMUM_SUPPORTED_DISTANCE} . * @param maxEdits must be >= 0 and <= {@link LevenshteinAutomata#MAXIMUM_SUPPORTED_DISTANCE} .
* @param transpositions <code>true</code> if transpositions should be treated as a primitive * @param transpositions <code>true</code> if transpositions should be treated as a primitive
* edit operation. If this is false, comparisons will implement the classic * edit operation. If this is false, comparisons will implement the classic
@ -153,9 +154,9 @@ public final class FuzzySuggester extends AnalyzingSuggester {
*/ */
public FuzzySuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer, public FuzzySuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer,
int options, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions, int options, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions,
int maxEdits, boolean transpositions, int nonFuzzyPrefix, boolean preservePositionIncrements, int maxEdits, boolean transpositions,
int minFuzzyLength, boolean unicodeAware) { int nonFuzzyPrefix, int minFuzzyLength, boolean unicodeAware) {
super(indexAnalyzer, queryAnalyzer, options, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions); super(indexAnalyzer, queryAnalyzer, options, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, preservePositionIncrements);
if (maxEdits < 0 || maxEdits > LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE) { if (maxEdits < 0 || maxEdits > LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE) {
throw new IllegalArgumentException("maxEdits must be between 0 and " + LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE); throw new IllegalArgumentException("maxEdits must be between 0 and " + LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE);
} }

39
lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
View File

@ -175,9 +175,8 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
mapping.put(title, Long.valueOf(randomWeight)); mapping.put(title, Long.valueOf(randomWeight));
} }
} }
AnalyzingSuggester analyzingSuggester = new AnalyzingSuggester(new MockAnalyzer(random()), new MockAnalyzer(random()),
AnalyzingSuggester analyzingSuggester = new AnalyzingSuggester(new MockAnalyzer(random())); AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, random().nextBoolean());
analyzingSuggester.setPreservePositionIncrements(random().nextBoolean());
boolean doPayloads = random().nextBoolean(); boolean doPayloads = random().nextBoolean();
if (doPayloads) { if (doPayloads) {
List<Input> keysAndPayloads = new ArrayList<>(); List<Input> keysAndPayloads = new ArrayList<>();
@ -214,8 +213,9 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
}; };
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET); Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
AnalyzingSuggester suggester = new AnalyzingSuggester(standard); AnalyzingSuggester suggester = new AnalyzingSuggester(standard, standard,
suggester.setPreservePositionIncrements(false); AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, false);
suggester.build(new InputArrayIterator(keys)); suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1); List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1);
@ -254,7 +254,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
int options = 0; int options = 0;
Analyzer a = new MockAnalyzer(random()); Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, options, 256, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, options, 256, -1, false);
suggester.build(new InputArrayIterator(keys)); suggester.build(new InputArrayIterator(keys));
// TODO: would be nice if "ab " would allow the test to // TODO: would be nice if "ab " would allow the test to
// pass, and more generally if the analyzer can know // pass, and more generally if the analyzer can know
@ -459,7 +459,8 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
public void testExactFirst() throws Exception { public void testExactFirst() throws Exception {
Analyzer a = getUnusualAnalyzer(); Analyzer a = getUnusualAnalyzer();
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1); int options = AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP;
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, options, 256, -1, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("x y", 1), new Input("x y", 1),
new Input("x y z", 3), new Input("x y z", 3),
@ -498,7 +499,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
public void testNonExactFirst() throws Exception { public void testNonExactFirst() throws Exception {
Analyzer a = getUnusualAnalyzer(); Analyzer a = getUnusualAnalyzer();
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("x y", 1), new Input("x y", 1),
@ -752,7 +753,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles); Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester suggester = new AnalyzingSuggester(a, a,
preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1); preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1, false);
if (doPayloads) { if (doPayloads) {
suggester.build(new InputArrayIterator(shuffle(payloadKeys))); suggester.build(new InputArrayIterator(shuffle(payloadKeys)));
} else { } else {
@ -873,7 +874,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception { public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {
Analyzer a = new MockAnalyzer(random()); Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 2, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 2, -1, false);
suggester.build(new InputArrayIterator(shuffle(new Input("a", 40), suggester.build(new InputArrayIterator(shuffle(new Input("a", 40),
new Input("a ", 50), new Input(" a", 60)))); new Input("a ", 50), new Input(" a", 60))));
@ -887,7 +888,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
public void testQueueExhaustion() throws Exception { public void testQueueExhaustion() throws Exception {
Analyzer a = new MockAnalyzer(random()); Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("a", 2), new Input("a", 2),
@ -903,7 +904,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
Analyzer a = new MockAnalyzer(random()); Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("a", 5), new Input("a", 5),
@ -968,7 +969,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
} }
}; };
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, false);
suggester.build(new InputArrayIterator(shuffle( suggester.build(new InputArrayIterator(shuffle(
new Input("hambone", 6), new Input("hambone", 6),
@ -1037,7 +1038,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
} }
}; };
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("a", 6), new Input("a", 6),
@ -1110,7 +1111,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
} }
}; };
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("a a", 50), new Input("a a", 50),
@ -1120,7 +1121,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
public void testDupSurfaceFormsMissingResults3() throws Exception { public void testDupSurfaceFormsMissingResults3() throws Exception {
Analyzer a = new MockAnalyzer(random()); Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("a a", 7), new Input("a a", 7),
new Input("a a", 7), new Input("a a", 7),
@ -1133,7 +1134,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
public void testEndingSpace() throws Exception { public void testEndingSpace() throws Exception {
Analyzer a = new MockAnalyzer(random()); Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("i love lucy", 7), new Input("i love lucy", 7),
new Input("isla de muerta", 8), new Input("isla de muerta", 8),
@ -1166,14 +1167,14 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
} }
}; };
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, 1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, 1, false);
suggester.build(new InputArrayIterator(new Input[] {new Input("a", 1)})); suggester.build(new InputArrayIterator(new Input[] {new Input("a", 1)}));
assertEquals("[a/1]", suggester.lookup("a", false, 1).toString()); assertEquals("[a/1]", suggester.lookup("a", false, 1).toString());
} }
public void testIllegalLookupArgument() throws Exception { public void testIllegalLookupArgument() throws Exception {
Analyzer a = new MockAnalyzer(random()); Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1); AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("а где Люси?", 7), new Input("а где Люси?", 7),
})); }));

22
lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
View File

@ -61,7 +61,7 @@ public class FuzzySuggesterTest extends LuceneTestCase {
} }
keys.add(new Input("foo bar boo far", 12)); keys.add(new Input("foo bar boo far", 12));
MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false); MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS, FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, false, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE); 0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE);
suggester.build(new InputArrayIterator(keys)); suggester.build(new InputArrayIterator(keys));
int numIters = atLeast(10); int numIters = atLeast(10);
@ -82,7 +82,7 @@ public class FuzzySuggesterTest extends LuceneTestCase {
} }
keys.add(new Input("фуу бар буу фар", 12)); keys.add(new Input("фуу бар буу фар", 12));
MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false); MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS, FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, false, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, true); 0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, true);
suggester.build(new InputArrayIterator(keys)); suggester.build(new InputArrayIterator(keys));
int numIters = atLeast(10); int numIters = atLeast(10);
@ -177,8 +177,8 @@ public class FuzzySuggesterTest extends LuceneTestCase {
}; };
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET); Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
FuzzySuggester suggester = new FuzzySuggester(standard); FuzzySuggester suggester = new FuzzySuggester(standard, standard, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, false, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
suggester.setPreservePositionIncrements(false); FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE);
suggester.build(new InputArrayIterator(keys)); suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1); List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1);
@ -208,7 +208,7 @@ public class FuzzySuggesterTest extends LuceneTestCase {
int options = 0; int options = 0;
Analyzer a = new MockAnalyzer(random()); Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, options, 256, -1, 1, true, 1, 3, false); FuzzySuggester suggester = new FuzzySuggester(a, a, options, 256, -1, false, 1, true, 1, 3, false);
suggester.build(new InputArrayIterator(keys)); suggester.build(new InputArrayIterator(keys));
// TODO: would be nice if "ab " would allow the test to // TODO: would be nice if "ab " would allow the test to
// pass, and more generally if the analyzer can know // pass, and more generally if the analyzer can know
@ -417,7 +417,7 @@ public class FuzzySuggesterTest extends LuceneTestCase {
public void testExactFirst() throws Exception { public void testExactFirst() throws Exception {
Analyzer a = getUnusualAnalyzer(); Analyzer a = getUnusualAnalyzer();
FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, 1, true, 1, 3, false); FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, false, 1, true, 1, 3, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("x y", 1), new Input("x y", 1),
new Input("x y z", 3), new Input("x y z", 3),
@ -456,7 +456,7 @@ public class FuzzySuggesterTest extends LuceneTestCase {
public void testNonExactFirst() throws Exception { public void testNonExactFirst() throws Exception {
Analyzer a = getUnusualAnalyzer(); Analyzer a = getUnusualAnalyzer();
FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, 1, true, 1, 3, false); FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, false, 1, true, 1, 3, false);
suggester.build(new InputArrayIterator(new Input[] { suggester.build(new InputArrayIterator(new Input[] {
new Input("x y", 1), new Input("x y", 1),
@ -683,7 +683,7 @@ public class FuzzySuggesterTest extends LuceneTestCase {
Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles); Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
FuzzySuggester suggester = new FuzzySuggester(a, a, FuzzySuggester suggester = new FuzzySuggester(a, a,
preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1, 1, false, 1, 3, unicodeAware); preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1, false, 1, false, 1, 3, unicodeAware);
suggester.build(new InputArrayIterator(keys)); suggester.build(new InputArrayIterator(keys));
for (String prefix : allPrefixes) { for (String prefix : allPrefixes) {
@ -823,7 +823,7 @@ public class FuzzySuggesterTest extends LuceneTestCase {
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception { public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {
Analyzer a = new MockAnalyzer(random()); Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, 0, 2, -1, 1, true, 1, 3, false); FuzzySuggester suggester = new FuzzySuggester(a, a, 0, 2, -1, false, 1, true, 1, 3, false);
List<Input> keys = Arrays.asList(new Input[] { List<Input> keys = Arrays.asList(new Input[] {
new Input("a", 40), new Input("a", 40),
@ -844,7 +844,7 @@ public class FuzzySuggesterTest extends LuceneTestCase {
public void testEditSeps() throws Exception { public void testEditSeps() throws Exception {
Analyzer a = new MockAnalyzer(random()); Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, FuzzySuggester.PRESERVE_SEP, 2, -1, 2, true, 1, 3, false); FuzzySuggester suggester = new FuzzySuggester(a, a, FuzzySuggester.PRESERVE_SEP, 2, -1, false, 2, true, 1, 3, false);
List<Input> keys = Arrays.asList(new Input[] { List<Input> keys = Arrays.asList(new Input[] {
new Input("foo bar", 40), new Input("foo bar", 40),
@ -958,7 +958,7 @@ public class FuzzySuggesterTest extends LuceneTestCase {
boolean transpositions = random().nextBoolean(); boolean transpositions = random().nextBoolean();
// TODO: test graph analyzers // TODO: test graph analyzers
// TODO: test exactFirst / preserveSep permutations // TODO: test exactFirst / preserveSep permutations
FuzzySuggester suggest = new FuzzySuggester(a, a, 0, 256, -1, maxEdits, transpositions, prefixLen, prefixLen, false); FuzzySuggester suggest = new FuzzySuggester(a, a, 0, 256, -1, false, maxEdits, transpositions, prefixLen, prefixLen, false);
if (VERBOSE) { if (VERBOSE) {
System.out.println("TEST: maxEdits=" + maxEdits + " prefixLen=" + prefixLen + " transpositions=" + transpositions + " num=" + NUM); System.out.println("TEST: maxEdits=" + maxEdits + " prefixLen=" + prefixLen + " transpositions=" + transpositions + " num=" + NUM);

12
solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingLookupFactory.java
View File

@ -64,6 +64,11 @@ public class AnalyzingLookupFactory extends LookupFactory {
*/ */
public static final String QUERY_ANALYZER = "suggestAnalyzerFieldType"; public static final String QUERY_ANALYZER = "suggestAnalyzerFieldType";
/**
* Whether position holes should appear in the automaton.
*/
public static final String PRESERVE_POSITION_INCREMENTS = "preservePositionIncrements";
/** /**
* File name for the automaton. * File name for the automaton.
* *
@ -106,9 +111,14 @@ public class AnalyzingLookupFactory extends LookupFactory {
int maxGraphExpansions = params.get(MAX_EXPANSIONS) != null int maxGraphExpansions = params.get(MAX_EXPANSIONS) != null
? Integer.parseInt(params.get(MAX_EXPANSIONS).toString()) ? Integer.parseInt(params.get(MAX_EXPANSIONS).toString())
: -1; : -1;
boolean preservePositionIncrements = params.get(PRESERVE_POSITION_INCREMENTS) != null
? Boolean.valueOf(params.get(PRESERVE_POSITION_INCREMENTS).toString())
: false;
return new AnalyzingSuggester(indexAnalyzer, queryAnalyzer, flags, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions); return new AnalyzingSuggester(indexAnalyzer, queryAnalyzer, flags, maxSurfaceFormsPerAnalyzedForm,
maxGraphExpansions, preservePositionIncrements);
} }
@Override @Override

10
solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java
View File

@ -106,6 +106,10 @@ public class FuzzyLookupFactory extends LookupFactory {
? Integer.parseInt(params.get(AnalyzingLookupFactory.MAX_EXPANSIONS).toString()) ? Integer.parseInt(params.get(AnalyzingLookupFactory.MAX_EXPANSIONS).toString())
: -1; : -1;
boolean preservePositionIncrements = params.get(AnalyzingLookupFactory.PRESERVE_POSITION_INCREMENTS) != null
? Boolean.valueOf(params.get(AnalyzingLookupFactory.PRESERVE_POSITION_INCREMENTS).toString())
: false;
int maxEdits = (params.get(MAX_EDITS) != null) int maxEdits = (params.get(MAX_EDITS) != null)
? Integer.parseInt(params.get(MAX_EDITS).toString()) ? Integer.parseInt(params.get(MAX_EDITS).toString())
: FuzzySuggester.DEFAULT_MAX_EDITS; : FuzzySuggester.DEFAULT_MAX_EDITS;
@ -127,9 +131,9 @@ public class FuzzyLookupFactory extends LookupFactory {
? Boolean.valueOf(params.get(UNICODE_AWARE).toString()) ? Boolean.valueOf(params.get(UNICODE_AWARE).toString())
: FuzzySuggester.DEFAULT_UNICODE_AWARE; : FuzzySuggester.DEFAULT_UNICODE_AWARE;
return new FuzzySuggester(indexAnalyzer, queryAnalyzer, options, return new FuzzySuggester(indexAnalyzer, queryAnalyzer, options, maxSurfaceFormsPerAnalyzedForm,
maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, maxEdits, maxGraphExpansions, preservePositionIncrements, maxEdits, transpositions, nonFuzzyPrefix,
transpositions, nonFuzzyPrefix, minFuzzyLength, unicodeAware); minFuzzyLength, unicodeAware);
} }
@Override @Override