Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[SOLR-3092] - updated TypeTokenFilterFactory to handle useWhiteList parameter 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1240081 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tommaso Teofili 2012-02-03 11:11:25 +00:00
parent 6d3bb736f3
commit 1b4c60816c
2 changed files with 23 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
solr/core/src/java/org/apache/solr/analysis/TypeTokenFilterFactory.java
View File

@ -27,31 +27,26 @@ import org.apache.solr.util.plugin.ResourceLoaderAware;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Factory class for {@link TypeTokenFilter}
* <pre class="prettyprint" >
* &lt;fieldType name="chars" class="solr.TextField" positionIncrementGap="100"&gt;
* &lt;analyzer&gt;
* &lt;tokenizer class="solr.StandardTokenizerFactory"/&gt;
* &lt;filter class="solr.TypeTokenFilterFactory" types="stoptypes.txt" enablePositionIncrements="true"/&gt;
* &lt;/analyzer&gt;
* &lt;analyzer&gt;
* &lt;tokenizer class="solr.StandardTokenizerFactory"/&gt;
* &lt;filter class="solr.TypeTokenFilterFactory" types="stoptypes.txt" enablePositionIncrements="true"
* useWhiteList="false"/&gt;
* &lt;/analyzer&gt;
* &lt;/fieldType&gt;</pre>
*/
public class TypeTokenFilterFactory extends BaseTokenFilterFactory implements ResourceLoaderAware {
@Override
public void init(Map<String, String> args) {
super.init(args);
}
@Override
public void inform(ResourceLoader loader) {
String stopTypesFiles = args.get("types");
enablePositionIncrements = getBoolean("enablePositionIncrements", false);
useWhitelist = getBoolean("useWhitelist", false);
if (stopTypesFiles != null) {
try {
List<String> files = StrUtils.splitFileNames(stopTypesFiles);
@ -70,6 +65,7 @@ public class TypeTokenFilterFactory extends BaseTokenFilterFactory implements Re
}
}
private boolean useWhitelist;
private Set<String> stopTypes;
private boolean enablePositionIncrements;
@ -83,6 +79,6 @@ public class TypeTokenFilterFactory extends BaseTokenFilterFactory implements Re
@Override
public TokenStream create(TokenStream input) {
return new TypeTokenFilter(enablePositionIncrements, input, stopTypes);
return new TypeTokenFilter(enablePositionIncrements, input, stopTypes, useWhitelist);
}
}

16
solr/core/src/test/org/apache/solr/analysis/TestTypeTokenFilterFactory.java
View File

@ -49,6 +49,7 @@ public class TestTypeTokenFilterFactory extends BaseTokenTestCase {
factory = new TypeTokenFilterFactory();
args.put("types", "stoptypes-1.txt, stoptypes-2.txt");
args.put("enablePositionIncrements", "false");
args.put("useWhitelist","true");
factory.init(args);
factory.inform(loader);
types = factory.getStopTypes();
@ -58,7 +59,7 @@ public class TestTypeTokenFilterFactory extends BaseTokenTestCase {
}
@Test
public void testCreation() throws Exception {
public void testCreationWithBlackList() throws Exception {
TypeTokenFilterFactory typeTokenFilterFactory = new TypeTokenFilterFactory();
Map<String, String> args = new HashMap<String, String>(DEFAULT_VERSION_PARAM);
args.put("types", "stoptypes-1.txt, stoptypes-2.txt");
@ -68,6 +69,19 @@ public class TestTypeTokenFilterFactory extends BaseTokenTestCase {
input.setIntValue(123);
typeTokenFilterFactory.create(input);
}
@Test
public void testCreationWithWhiteList() throws Exception {
TypeTokenFilterFactory typeTokenFilterFactory = new TypeTokenFilterFactory();
Map<String, String> args = new HashMap<String, String>(DEFAULT_VERSION_PARAM);
args.put("types", "stoptypes-1.txt, stoptypes-2.txt");
args.put("enablePositionIncrements", "false");
args.put("useWhitelist","true");
typeTokenFilterFactory.init(args);
NumericTokenStream input = new NumericTokenStream();
input.setIntValue(123);
typeTokenFilterFactory.create(input);
}
@Test
public void testMissingTypesParameter() throws Exception {