mirror of https://github.com/apache/lucene.git
SOLR-468: Changed semantics of CapitalizationFilterFactory to keep the original token, not the value in the map
git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@630572 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
8cf3175518
commit
ac58a1a4dd
|
@ -106,6 +106,8 @@ New Features
|
|||
17. SOLR-248: Added CapitalizationFilterFactory that creates tokens with
|
||||
normalized capitalization. This filter is useful for facet display,
|
||||
but will not work with a prefix query. (ryan)
|
||||
SOLR-468: Change to the semantics to keep the original token, not the token in the Map. Also switched to
|
||||
use Lucene's new reusable token capabilities. (gsingers)
|
||||
|
||||
18. SOLR-307: Added NGramFilterFactory and EdgeNGramFilterFactory.
|
||||
(Thomas Peuss via Otis Gospodnetic)
|
||||
|
|
|
@ -17,199 +17,226 @@
|
|||
|
||||
package org.apache.solr.analysis;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.StringTokenizer;
|
||||
|
||||
import org.apache.lucene.analysis.CharArraySet;
|
||||
import org.apache.lucene.analysis.Token;
|
||||
import org.apache.lucene.analysis.TokenFilter;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.solr.core.SolrConfig;
|
||||
import org.apache.solr.analysis.BaseTokenFilterFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.StringTokenizer;
|
||||
|
||||
/**
|
||||
* A filter to apply normal capitalization rules to Tokens. It will make the first letter
|
||||
* capital and the rest lower case.
|
||||
*
|
||||
* capital and the rest lower case.
|
||||
* <p/>
|
||||
* This filter is particularly useful to build nice looking facet parameters. This filter
|
||||
* is not appropriate if you intend to use a prefix query.
|
||||
*
|
||||
* The factory takes parameters:
|
||||
* "onlyFirstWord" - should each word be capitalized or all of the words?
|
||||
* "keep" - a keep word list. Each word that should be kept separated by whitespace.
|
||||
* <p/>
|
||||
* The factory takes parameters:<br/>
|
||||
* "onlyFirstWord" - should each word be capitalized or all of the words?<br/>
|
||||
* "keep" - a keep word list. Each word that should be kept separated by whitespace.<br/>
|
||||
* "keepIgnoreCase - true or false. If true, the keep list will be considered case-insensitive.
|
||||
* "forceFirstLetter" - Force the first letter to be capitalized even if it is in the keep list<br/>
|
||||
* "okPrefix" - do not change word capitalization if a word begins with something in this list.
|
||||
* for example if "McK" is on the okPrefix list, the word "McKinley" should not be changed to
|
||||
* "Mckinley"
|
||||
* "minWordLength" - how long the word needs to be to get capitalization applied. If the
|
||||
* minWordLength is 3, "and" > "And" but "or" stays "or"
|
||||
* for example if "McK" is on the okPrefix list, the word "McKinley" should not be changed to
|
||||
* "Mckinley"<br/>
|
||||
* "minWordLength" - how long the word needs to be to get capitalization applied. If the
|
||||
* minWordLength is 3, "and" > "And" but "or" stays "or"<br/>
|
||||
* "maxWordCount" - if the token contains more then maxWordCount words, the capitalization is
|
||||
* assumed to be correct.
|
||||
*
|
||||
* @since solr 1.3
|
||||
* assumed to be correct.<br/>
|
||||
*
|
||||
* @version $Id$
|
||||
* @since solr 1.3
|
||||
*/
|
||||
public class CapitalizationFilterFactory extends BaseTokenFilterFactory
|
||||
{
|
||||
public class CapitalizationFilterFactory extends BaseTokenFilterFactory {
|
||||
public static final int DEFAULT_MAX_WORD_COUNT = Integer.MAX_VALUE;
|
||||
public static final String KEEP = "keep";
|
||||
public static final String KEEP_IGNORE_CASE = "keepIgnoreCase";
|
||||
public static final String OK_PREFIX = "okPrefix";
|
||||
public static final String MIN_WORD_LENGTH = "minWordLength";
|
||||
public static final String MAX_WORD_COUNT = "maxWordCount";
|
||||
public static final String MAX_TOKEN_LENGTH = "maxTokenLength";
|
||||
public static final String ONLY_FIRST_WORD = "onlyFirstWord";
|
||||
public static final String FORCE_FIRST_LETTER = "forceFirstLetter";
|
||||
|
||||
Map<String,String> keep = new HashMap<String, String>(); // not synchronized because it is only initialized once
|
||||
|
||||
Collection<String> okPrefix = new ArrayList<String>(); // for Example: McK
|
||||
|
||||
|
||||
//Map<String,String> keep = new HashMap<String, String>(); // not synchronized because it is only initialized once
|
||||
CharArraySet keep;
|
||||
|
||||
Collection<char[]> okPrefix = Collections.emptyList(); // for Example: McK
|
||||
|
||||
int minWordLength = 0; // don't modify capitalization for words shorter then this
|
||||
int maxWordCount = Integer.MAX_VALUE;
|
||||
int maxTokenLength = Integer.MAX_VALUE;
|
||||
int maxWordCount = DEFAULT_MAX_WORD_COUNT;
|
||||
int maxTokenLength = DEFAULT_MAX_WORD_COUNT;
|
||||
boolean onlyFirstWord = true;
|
||||
boolean forceFirstLetter = true; // make sure the first letter is capitol even if it is in the keep list
|
||||
|
||||
|
||||
@Override
|
||||
public void init(Map<String,String> args) {
|
||||
super.init( args );
|
||||
|
||||
String k = args.get( KEEP );
|
||||
if( k != null ) {
|
||||
StringTokenizer st = new StringTokenizer( k );
|
||||
while( st.hasMoreTokens() ) {
|
||||
public void init(Map<String, String> args) {
|
||||
super.init(args);
|
||||
|
||||
String k = args.get(KEEP);
|
||||
if (k != null) {
|
||||
StringTokenizer st = new StringTokenizer(k);
|
||||
boolean ignoreCase = false;
|
||||
String ignoreStr = args.get(KEEP_IGNORE_CASE);
|
||||
if ("true".equalsIgnoreCase(ignoreStr)) {
|
||||
ignoreCase = true;
|
||||
}
|
||||
keep = new CharArraySet(10, ignoreCase);
|
||||
while (st.hasMoreTokens()) {
|
||||
k = st.nextToken().trim();
|
||||
keep.put( k.toUpperCase(), k );
|
||||
keep.add(k.toCharArray());
|
||||
}
|
||||
}
|
||||
|
||||
k = args.get( OK_PREFIX );
|
||||
if( k != null ) {
|
||||
StringTokenizer st = new StringTokenizer( k );
|
||||
while( st.hasMoreTokens() ) {
|
||||
okPrefix.add( st.nextToken().trim() );
|
||||
|
||||
k = args.get(OK_PREFIX);
|
||||
if (k != null) {
|
||||
okPrefix = new ArrayList<char[]>();
|
||||
StringTokenizer st = new StringTokenizer(k);
|
||||
while (st.hasMoreTokens()) {
|
||||
okPrefix.add(st.nextToken().trim().toCharArray());
|
||||
}
|
||||
}
|
||||
|
||||
k = args.get( MIN_WORD_LENGTH );
|
||||
if( k != null ) {
|
||||
minWordLength = Integer.valueOf( k );
|
||||
|
||||
k = args.get(MIN_WORD_LENGTH);
|
||||
if (k != null) {
|
||||
minWordLength = Integer.valueOf(k);
|
||||
}
|
||||
|
||||
k = args.get( MAX_WORD_COUNT );
|
||||
if( k != null ) {
|
||||
maxWordCount = Integer.valueOf( k );
|
||||
k = args.get(MAX_WORD_COUNT);
|
||||
if (k != null) {
|
||||
maxWordCount = Integer.valueOf(k);
|
||||
}
|
||||
|
||||
k = args.get( MAX_TOKEN_LENGTH );
|
||||
if( k != null ) {
|
||||
maxTokenLength = Integer.valueOf( k );
|
||||
k = args.get(MAX_TOKEN_LENGTH);
|
||||
if (k != null) {
|
||||
maxTokenLength = Integer.valueOf(k);
|
||||
}
|
||||
|
||||
k = args.get( ONLY_FIRST_WORD );
|
||||
if( k != null ) {
|
||||
onlyFirstWord = Boolean.valueOf( k );
|
||||
k = args.get(ONLY_FIRST_WORD);
|
||||
if (k != null) {
|
||||
onlyFirstWord = Boolean.valueOf(k);
|
||||
}
|
||||
|
||||
k = args.get( FORCE_FIRST_LETTER );
|
||||
if( k != null ) {
|
||||
forceFirstLetter = Boolean.valueOf( k );
|
||||
k = args.get(FORCE_FIRST_LETTER);
|
||||
if (k != null) {
|
||||
forceFirstLetter = Boolean.valueOf(k);
|
||||
}
|
||||
}
|
||||
|
||||
public String processWord( String w, int wordCount )
|
||||
{
|
||||
if( w.length() < 1 ) {
|
||||
return w;
|
||||
|
||||
|
||||
public void processWord(char[] buffer, int offset, int length, int wordCount) {
|
||||
if (length < 1) {
|
||||
return;
|
||||
}
|
||||
if( onlyFirstWord && wordCount > 0 ) {
|
||||
return w.toLowerCase();
|
||||
}
|
||||
|
||||
String k = keep.get( w.toUpperCase() );
|
||||
if( k != null ) {
|
||||
if( wordCount == 0 && forceFirstLetter && Character.isLowerCase( k.charAt(0) ) ) {
|
||||
return Character.toUpperCase( k.charAt(0) ) + k.substring( 1 );
|
||||
if (onlyFirstWord && wordCount > 0) {
|
||||
for (int i = 0; i < length; i++) {
|
||||
buffer[offset + i] = Character.toLowerCase(buffer[offset + i]);
|
||||
|
||||
}
|
||||
return k;
|
||||
return;
|
||||
}
|
||||
if( w.length() < minWordLength ) {
|
||||
return w;
|
||||
|
||||
if (keep.contains(buffer, offset, length)) {
|
||||
if (wordCount == 0 && forceFirstLetter) {
|
||||
buffer[offset] = Character.toUpperCase(buffer[offset]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
for( String prefix : okPrefix ) {
|
||||
if( w.startsWith( prefix ) ) {
|
||||
return w;
|
||||
if (length < minWordLength) {
|
||||
return;
|
||||
}
|
||||
for (char[] prefix : okPrefix) {
|
||||
if (length >= prefix.length) { //don't bother checking if the buffer length is less than the prefix
|
||||
boolean match = true;
|
||||
for (int i = 0; i < prefix.length; i++) {
|
||||
if (prefix[i] != buffer[offset + i]) {
|
||||
match = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (match == true) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// We know it has at least one character
|
||||
char[] chars = w.toCharArray();
|
||||
/*char[] chars = w.toCharArray();
|
||||
StringBuilder word = new StringBuilder( w.length() );
|
||||
word.append( Character.toUpperCase( chars[0] ) );
|
||||
for( int i=1; i<chars.length; i++ ) {
|
||||
word.append( Character.toLowerCase( chars[i] ) );
|
||||
word.append( Character.toUpperCase( chars[0] ) );*/
|
||||
buffer[offset] = Character.toUpperCase(buffer[offset]);
|
||||
|
||||
for (int i = 1; i < length; i++) {
|
||||
buffer[offset + i] = Character.toLowerCase(buffer[offset + i]);
|
||||
}
|
||||
return word.toString();
|
||||
//return word.toString();
|
||||
}
|
||||
|
||||
|
||||
public CapitalizationFilter create(TokenStream input) {
|
||||
return new CapitalizationFilter(input,this);
|
||||
return new CapitalizationFilter(input, this);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* This relies on the Factory so that the difficult stuff does not need to be
|
||||
* re-initialized each time the filter runs.
|
||||
*
|
||||
* <p/>
|
||||
* This is package protected since it is not useful without the Factory
|
||||
*/
|
||||
class CapitalizationFilter extends TokenFilter
|
||||
{
|
||||
class CapitalizationFilter extends TokenFilter {
|
||||
protected final CapitalizationFilterFactory factory;
|
||||
|
||||
public CapitalizationFilter(TokenStream in, final CapitalizationFilterFactory factory ) {
|
||||
|
||||
public CapitalizationFilter(TokenStream in, final CapitalizationFilterFactory factory) {
|
||||
super(in);
|
||||
this.factory = factory;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public final Token next() throws IOException {
|
||||
|
||||
Token t = input.next();
|
||||
if( t != null ) {
|
||||
String s = t.termText();
|
||||
if( s.length() < factory.maxTokenLength ) {
|
||||
public Token next(Token token) throws IOException {
|
||||
Token t = input.next(token);
|
||||
if (t != null) {
|
||||
|
||||
char[] termBuffer = t.termBuffer();
|
||||
int termBufferLength = t.termLength();
|
||||
char[] backup = null;
|
||||
if (factory.maxWordCount < CapitalizationFilterFactory.DEFAULT_MAX_WORD_COUNT) {
|
||||
//make a backup in case we exceed the word count
|
||||
System.arraycopy(termBuffer, 0, backup, 0, termBufferLength);
|
||||
}
|
||||
if (termBuffer.length < factory.maxTokenLength) {
|
||||
int wordCount = 0;
|
||||
|
||||
StringBuilder word = new StringBuilder( s.length() );
|
||||
StringBuilder text = new StringBuilder( s.length() );
|
||||
for( char c : s.toCharArray() ) {
|
||||
if( c <= ' ' || c == '.' ) {
|
||||
if( word.length() > 0 ) {
|
||||
text.append( factory.processWord( word.toString(), wordCount++ ) );
|
||||
word.setLength( 0 );
|
||||
int lastWordStart = 0;
|
||||
for (int i = 0; i < termBufferLength; i++) {
|
||||
char c = termBuffer[i];
|
||||
if (c <= ' ' || c == '.') {
|
||||
int len = i - lastWordStart;
|
||||
if (len > 0) {
|
||||
factory.processWord(termBuffer, lastWordStart, len, wordCount++);
|
||||
lastWordStart = i + 1;
|
||||
i++;
|
||||
}
|
||||
text.append( c );
|
||||
}
|
||||
else {
|
||||
word.append( c );
|
||||
}
|
||||
}
|
||||
|
||||
// Add the last word
|
||||
if( word.length() > 0 ) {
|
||||
text.append( factory.processWord( word.toString(), wordCount++ ) );
|
||||
|
||||
// process the last word
|
||||
if (lastWordStart < termBuffer.length) {
|
||||
factory.processWord(termBuffer, lastWordStart, termBuffer.length - lastWordStart, wordCount++);
|
||||
}
|
||||
|
||||
if( wordCount <= factory.maxWordCount ) {
|
||||
t.setTermText( text.toString() );
|
||||
|
||||
if (wordCount > factory.maxWordCount) {
|
||||
t.setTermBuffer(backup, 0, termBufferLength);
|
||||
}
|
||||
}
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
@ -34,14 +34,33 @@ public class TestCapitalizationFilter extends BaseTokenTestCase {
|
|||
|
||||
CapitalizationFilterFactory factory = new CapitalizationFilterFactory();
|
||||
factory.init( args );
|
||||
char[] termBuffer;
|
||||
termBuffer = "kiTTEN".toCharArray();
|
||||
factory.processWord(termBuffer, 0, termBuffer.length, 0 );
|
||||
assertEquals( "Kitten", new String(termBuffer, 0, termBuffer.length));
|
||||
|
||||
assertEquals( "Kitten", factory.processWord( "kiTTEN", 0 ) );
|
||||
factory.forceFirstLetter = true;
|
||||
assertEquals( "And", factory.processWord( "AnD", 0 ) ); // first is forced
|
||||
|
||||
termBuffer = "and".toCharArray();
|
||||
factory.processWord(termBuffer, 0, termBuffer.length, 0 );
|
||||
assertEquals( "And", new String(termBuffer, 0, termBuffer.length));//first is forced
|
||||
|
||||
termBuffer = "AnD".toCharArray();
|
||||
factory.processWord(termBuffer, 0, termBuffer.length, 0 );
|
||||
assertEquals( "And", new String(termBuffer, 0, termBuffer.length));//first is forced, but it's not a keep word, either
|
||||
|
||||
factory.forceFirstLetter = false;
|
||||
assertEquals( "and", factory.processWord( "AnD", 0 ) ); // first is forced
|
||||
termBuffer = "AnD".toCharArray();
|
||||
factory.processWord(termBuffer, 0, termBuffer.length, 0 );
|
||||
assertEquals( "And", new String(termBuffer, 0, termBuffer.length)); //first is not forced, but it's not a keep word, either
|
||||
|
||||
factory.forceFirstLetter = true;
|
||||
assertEquals( "BIG", factory.processWord( "big", 0 ) );
|
||||
termBuffer = "big".toCharArray();
|
||||
factory.processWord(termBuffer, 0, termBuffer.length, 0 );
|
||||
assertEquals( "Big", new String(termBuffer, 0, termBuffer.length));
|
||||
termBuffer = "BIG".toCharArray();
|
||||
factory.processWord(termBuffer, 0, termBuffer.length, 0 );
|
||||
assertEquals( "BIG", new String(termBuffer, 0, termBuffer.length));
|
||||
|
||||
String out = tsToString( factory.create( new IterTokenStream( "Hello thEre my Name is Ryan" ) ) );
|
||||
assertEquals( "Hello there my name is ryan", out );
|
||||
|
@ -74,7 +93,27 @@ public class TestCapitalizationFilter extends BaseTokenTestCase {
|
|||
assertEquals( "1st 2nd Third", out );
|
||||
|
||||
factory.forceFirstLetter = true;
|
||||
out = tsToString( factory.create( new IterTokenStream( "the The" ) ) );
|
||||
assertEquals( "The the", out );
|
||||
out = tsToString( factory.create( new IterTokenStream( "the The the" ) ) );
|
||||
assertEquals( "The The the", out );
|
||||
}
|
||||
|
||||
public void testKeepIgnoreCase() throws Exception {
|
||||
Map<String,String> args = new HashMap<String, String>();
|
||||
args.put( CapitalizationFilterFactory.KEEP, "kitten" );
|
||||
args.put( CapitalizationFilterFactory.KEEP_IGNORE_CASE, "true" );
|
||||
args.put( CapitalizationFilterFactory.ONLY_FIRST_WORD, "true" );
|
||||
|
||||
CapitalizationFilterFactory factory = new CapitalizationFilterFactory();
|
||||
factory.init( args );
|
||||
char[] termBuffer;
|
||||
termBuffer = "kiTTEN".toCharArray();
|
||||
factory.forceFirstLetter = true;
|
||||
factory.processWord(termBuffer, 0, termBuffer.length, 0 );
|
||||
assertEquals( "KiTTEN", new String(termBuffer, 0, termBuffer.length));
|
||||
|
||||
factory.forceFirstLetter = false;
|
||||
termBuffer = "kiTTEN".toCharArray();
|
||||
factory.processWord(termBuffer, 0, termBuffer.length, 0 );
|
||||
assertEquals( "kiTTEN", new String(termBuffer, 0, termBuffer.length));
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue