mirror of https://github.com/apache/lucene.git
- Fixed a bug with substCount variable which was not being reset.
- Re-formatted the whole class to match the rest of the code more closely (still differs, but... :)). git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@149680 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Otis Gospodnetic
parent
9bd57fb82a
commit
3e7f46010e
|
|
@ -1,7 +1,7 @@
|
||||||
package org.apache.lucene.analysis.de;
|
package org.apache.lucene.analysis.de;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A stemmer for german words. The algorithm is based on the report
|
* A stemmer for German words. The algorithm is based on the report
|
||||||
* "A Fast and Simple Stemming Algorithm for German Words" by Jörg
|
* "A Fast and Simple Stemming Algorithm for German Words" by Jörg
|
||||||
* Caumanns (joerg.caumanns@isst.fhg.de).
|
* Caumanns (joerg.caumanns@isst.fhg.de).
|
||||||
*
|
*
|
||||||
|
|
@ -11,277 +11,277 @@ package org.apache.lucene.analysis.de;
|
||||||
|
|
||||||
public class GermanStemmer {
|
public class GermanStemmer {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Buffer for the terms while stemming them.
|
* Buffer for the terms while stemming them.
|
||||||
*/
|
*/
|
||||||
private StringBuffer sb = new StringBuffer();
|
private StringBuffer sb = new StringBuffer();
|
||||||
/**
|
/**
|
||||||
* Indicates if a term is handled as a noun.
|
* Indicates if a term is handled as a noun.
|
||||||
*/
|
*/
|
||||||
private boolean uppercase = false;
|
private boolean uppercase = false;
|
||||||
/**
|
/**
|
||||||
* Amount of characters that are removed with <tt>substitute()</tt> while stemming.
|
* Amount of characters that are removed with <tt>substitute()</tt> while stemming.
|
||||||
*/
|
*/
|
||||||
private int substCount = 0;
|
private int substCount = 0;
|
||||||
|
|
||||||
public GermanStemmer() {
|
public GermanStemmer() {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stemms the given term to an unique <tt>discriminator</tt>.
|
||||||
|
*
|
||||||
|
* @param term The term that should be stemmed.
|
||||||
|
* @return Discriminator for <tt>term</tt>
|
||||||
|
*/
|
||||||
|
protected String stem( String term ) {
|
||||||
|
if ( !isStemmable( term ) ) {
|
||||||
|
return term;
|
||||||
}
|
}
|
||||||
|
// Mark a possible noun.
|
||||||
/**
|
if ( Character.isUpperCase( term.charAt( 0 ) ) ) {
|
||||||
* Stemms the given term to an unique <tt>discriminator</tt>.
|
uppercase = true;
|
||||||
*
|
}
|
||||||
* @param term The term that should be stemmed.
|
else {
|
||||||
* @return Discriminator for <tt>term</tt>
|
uppercase = false;
|
||||||
*/
|
}
|
||||||
protected String stem( String term ) {
|
// Use lowercase for medium stemming.
|
||||||
if ( !isStemmable( term ) ) {
|
term = term.toLowerCase();
|
||||||
return term;
|
// Reset the StringBuffer.
|
||||||
|
sb.delete( 0, sb.length() );
|
||||||
|
sb.insert( 0, term );
|
||||||
|
sb = substitute( sb );
|
||||||
|
// Nouns have only seven possible suffixes.
|
||||||
|
if ( uppercase && sb.length() > 3 ) {
|
||||||
|
if ( sb.substring( sb.length() - 3, sb.length() ).equals( "ern" ) ) {
|
||||||
|
sb.delete( sb.length() - 3, sb.length() );
|
||||||
|
}
|
||||||
|
else if ( sb.substring( sb.length() - 2, sb.length() ).equals( "en" ) ) {
|
||||||
|
sb.delete( sb.length() - 2, sb.length() );
|
||||||
|
}
|
||||||
|
else if ( sb.substring( sb.length() - 2, sb.length() ).equals( "er" ) ) {
|
||||||
|
sb.delete( sb.length() - 2, sb.length() );
|
||||||
|
}
|
||||||
|
else if ( sb.substring( sb.length() - 2, sb.length() ).equals( "es" ) ) {
|
||||||
|
sb.delete( sb.length() - 2, sb.length() );
|
||||||
|
}
|
||||||
|
else if ( sb.charAt( sb.length() - 1 ) == 'e' ) {
|
||||||
|
sb.deleteCharAt( sb.length() - 1 );
|
||||||
|
}
|
||||||
|
else if ( sb.charAt( sb.length() - 1 ) == 'n' ) {
|
||||||
|
sb.deleteCharAt( sb.length() - 1 );
|
||||||
|
}
|
||||||
|
else if ( sb.charAt( sb.length() - 1 ) == 's' ) {
|
||||||
|
sb.deleteCharAt( sb.length() - 1 );
|
||||||
|
}
|
||||||
|
// Additional step for female plurals of professions and inhabitants.
|
||||||
|
if ( sb.length() > 5 && sb.substring( sb.length() - 3, sb.length() ).equals( "erin*" ) ) {
|
||||||
|
sb.deleteCharAt( sb.length() -1 );
|
||||||
|
}
|
||||||
|
// Additional step for irregular plural nouns like "Matrizen -> Matrix".
|
||||||
|
if ( sb.charAt( sb.length() - 1 ) == ( 'z' ) ) {
|
||||||
|
sb.setCharAt( sb.length() - 1, 'x' );
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Strip the 7 "base" suffixes: "e", "s", "n", "t", "em", "er", "nd" from all
|
||||||
|
// other terms. Adjectives, Verbs and Adverbs have a total of 52 different
|
||||||
|
// possible suffixes, stripping only the characters from they are build
|
||||||
|
// does mostly the same
|
||||||
|
else {
|
||||||
|
// Strip base suffixes as long as enough characters remain.
|
||||||
|
boolean doMore = true;
|
||||||
|
while ( sb.length() > 3 && doMore ) {
|
||||||
|
if ( ( sb.length() + substCount > 5 ) && sb.substring( sb.length() - 2, sb.length() ).equals( "nd" ) ) {
|
||||||
|
sb.delete( sb.length() - 2, sb.length() );
|
||||||
}
|
}
|
||||||
// Mark a possible noun.
|
else if ( ( sb.length() + substCount > 4 ) && sb.substring( sb.length() - 2, sb.length() ).equals( "er" ) ) {
|
||||||
if ( Character.isUpperCase( term.charAt( 0 ) ) ) {
|
sb.delete( sb.length() - 2, sb.length() );
|
||||||
uppercase = true;
|
}
|
||||||
|
else if ( ( sb.length() + substCount > 4 ) && sb.substring( sb.length() - 2, sb.length() ).equals( "em" ) ) {
|
||||||
|
sb.delete( sb.length() - 2, sb.length() );
|
||||||
|
}
|
||||||
|
else if ( sb.charAt( sb.length() - 1 ) == 't' ) {
|
||||||
|
sb.deleteCharAt( sb.length() - 1 );
|
||||||
|
}
|
||||||
|
else if ( sb.charAt( sb.length() - 1 ) == 'n' ) {
|
||||||
|
sb.deleteCharAt( sb.length() - 1 );
|
||||||
|
}
|
||||||
|
else if ( sb.charAt( sb.length() - 1 ) == 's' ) {
|
||||||
|
sb.deleteCharAt( sb.length() - 1 );
|
||||||
|
}
|
||||||
|
else if ( sb.charAt( sb.length() - 1 ) == 'e' ) {
|
||||||
|
sb.deleteCharAt( sb.length() - 1 );
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
uppercase = false;
|
doMore = false;
|
||||||
}
|
}
|
||||||
// Use lowercase for medium stemming.
|
}
|
||||||
term = term.toLowerCase();
|
}
|
||||||
// Reset the StringBuffer.
|
sb = resubstitute( sb );
|
||||||
sb.delete( 0, sb.length() );
|
if ( !uppercase ) {
|
||||||
sb.insert( 0, term );
|
sb = removeParticleDenotion( sb );
|
||||||
sb = substitute( sb );
|
}
|
||||||
// Nouns have only seven possible suffixes.
|
return sb.toString();
|
||||||
if ( uppercase && sb.length() > 3 ) {
|
}
|
||||||
if ( sb.substring( sb.length() - 3, sb.length() ).equals( "ern" ) ) {
|
|
||||||
sb.delete( sb.length() - 3, sb.length() );
|
/**
|
||||||
}
|
* Removes a particle denotion ("ge") from a term, but only if at least 3
|
||||||
else if ( sb.substring( sb.length() - 2, sb.length() ).equals( "en" ) ) {
|
* characters will remain.
|
||||||
sb.delete( sb.length() - 2, sb.length() );
|
*
|
||||||
}
|
* @return The term without particle denotion, if there was one.
|
||||||
else if ( sb.substring( sb.length() - 2, sb.length() ).equals( "er" ) ) {
|
*/
|
||||||
sb.delete( sb.length() - 2, sb.length() );
|
private StringBuffer removeParticleDenotion( StringBuffer buffer ) {
|
||||||
}
|
for ( int c = 0; c < buffer.length(); c++ ) {
|
||||||
else if ( sb.substring( sb.length() - 2, sb.length() ).equals( "es" ) ) {
|
// Strip from the beginning of the string to the "ge" inclusive
|
||||||
sb.delete( sb.length() - 2, sb.length() );
|
if ( c < ( buffer.length() - 4 ) && buffer.charAt( c ) == 'g' && buffer.charAt ( c + 1 ) == 'e' ) {
|
||||||
}
|
buffer.delete( 0, c + 2 );
|
||||||
else if ( sb.charAt( sb.length() - 1 ) == 'e' ) {
|
}
|
||||||
sb.deleteCharAt( sb.length() - 1 );
|
}
|
||||||
}
|
return sb;
|
||||||
else if ( sb.charAt( sb.length() - 1 ) == 'n' ) {
|
}
|
||||||
sb.deleteCharAt( sb.length() - 1 );
|
|
||||||
}
|
/**
|
||||||
else if ( sb.charAt( sb.length() - 1 ) == 's' ) {
|
* Do some substitutions for the term to reduce overstemming:
|
||||||
sb.deleteCharAt( sb.length() - 1 );
|
*
|
||||||
}
|
* - Substitute Umlauts with their corresponding vowel: äöü -> aou,
|
||||||
// Additional step for female plurals of professions and inhabitants.
|
* "ß" is substituted by "ss"
|
||||||
if ( sb.length() > 5 && sb.substring( sb.length() - 3, sb.length() ).equals( "erin*" ) ) {
|
* - Substitute a second char of an pair of equal characters with
|
||||||
sb.deleteCharAt( sb.length() -1 );
|
* an asterisk: ?? -> ?*
|
||||||
}
|
* - Substitute some common character combinations with a token:
|
||||||
// Additional step for irregular plural nouns like "Matrizen -> Matrix".
|
* sch/ch/ei/ie/ig/st -> $/§/%/&/#/!
|
||||||
if ( sb.charAt( sb.length() - 1 ) == ( 'z' ) ) {
|
*
|
||||||
sb.setCharAt( sb.length() - 1, 'x' );
|
* @return The term with all needed substitutions.
|
||||||
}
|
*/
|
||||||
|
private StringBuffer substitute( StringBuffer buffer ) {
|
||||||
|
substCount = 0;
|
||||||
|
for ( int c = 0; c < buffer.length(); c++ ) {
|
||||||
|
// Replace the second char of a pair of the equal characters with an asterisk
|
||||||
|
if ( c > 0 && buffer.charAt( c ) == buffer.charAt ( c - 1 ) ) {
|
||||||
|
buffer.setCharAt( c, '*' );
|
||||||
|
}
|
||||||
|
// Substitute Umlauts.
|
||||||
|
else if ( buffer.charAt( c ) == 'ä' ) {
|
||||||
|
buffer.setCharAt( c, 'a' );
|
||||||
|
}
|
||||||
|
else if ( buffer.charAt( c ) == 'ö' ) {
|
||||||
|
buffer.setCharAt( c, 'o' );
|
||||||
|
}
|
||||||
|
else if ( buffer.charAt( c ) == 'ü' ) {
|
||||||
|
buffer.setCharAt( c, 'u' );
|
||||||
|
}
|
||||||
|
// Take care that at least one character is left left side from the current one
|
||||||
|
if ( c < buffer.length() - 1 ) {
|
||||||
|
if ( buffer.charAt( c ) == 'ß' ) {
|
||||||
|
buffer.setCharAt( c, 's' );
|
||||||
|
buffer.insert( c + 1, 's' );
|
||||||
|
substCount++;
|
||||||
}
|
}
|
||||||
// Strip the 7 "base" suffixes: "e", "s", "n", "t", "em", "er", "nd" from all
|
// Masking several common character combinations with an token
|
||||||
// other terms. Adjectives, Verbs and Adverbs have a total of 52 different
|
else if ( ( c < buffer.length() - 2 ) && buffer.charAt( c ) == 's' && buffer.charAt( c + 1 ) == 'c' && buffer.charAt( c + 2 ) == 'h' ) {
|
||||||
// possible suffixes, stripping only the characters from they are build
|
buffer.setCharAt( c, '$' );
|
||||||
// does mostly the same
|
buffer.delete( c + 1, c + 3 );
|
||||||
|
substCount =+ 2;
|
||||||
|
}
|
||||||
|
else if ( buffer.charAt( c ) == 'c' && buffer.charAt( c + 1 ) == 'h' ) {
|
||||||
|
buffer.setCharAt( c, '§' );
|
||||||
|
buffer.deleteCharAt( c + 1 );
|
||||||
|
substCount++;
|
||||||
|
}
|
||||||
|
else if ( buffer.charAt( c ) == 'e' && buffer.charAt( c + 1 ) == 'i' ) {
|
||||||
|
buffer.setCharAt( c, '%' );
|
||||||
|
buffer.deleteCharAt( c + 1 );
|
||||||
|
substCount++;
|
||||||
|
}
|
||||||
|
else if ( buffer.charAt( c ) == 'i' && buffer.charAt( c + 1 ) == 'e' ) {
|
||||||
|
buffer.setCharAt( c, '&' );
|
||||||
|
buffer.deleteCharAt( c + 1 );
|
||||||
|
substCount++;
|
||||||
|
}
|
||||||
|
else if ( buffer.charAt( c ) == 'i' && buffer.charAt( c + 1 ) == 'g' ) {
|
||||||
|
buffer.setCharAt( c, '#' );
|
||||||
|
buffer.deleteCharAt( c + 1 );
|
||||||
|
substCount++;
|
||||||
|
}
|
||||||
|
else if ( buffer.charAt( c ) == 's' && buffer.charAt( c + 1 ) == 't' ) {
|
||||||
|
buffer.setCharAt( c, '!' );
|
||||||
|
buffer.deleteCharAt( c + 1 );
|
||||||
|
substCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks a term if it can be processed correctly.
|
||||||
|
*
|
||||||
|
* @return true if, and only if, the given term consists in letters.
|
||||||
|
*/
|
||||||
|
private boolean isStemmable( String term ) {
|
||||||
|
boolean upper = false;
|
||||||
|
int first = -1;
|
||||||
|
for ( int c = 0; c < term.length(); c++ ) {
|
||||||
|
// Discard terms that contain non-letter characters.
|
||||||
|
if ( !Character.isLetter( term.charAt( c ) ) ) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// Discard terms that contain multiple uppercase letters.
|
||||||
|
if ( Character.isUpperCase( term.charAt( c ) ) ) {
|
||||||
|
if ( upper ) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// First encountered uppercase letter, set flag and save
|
||||||
|
// position.
|
||||||
else {
|
else {
|
||||||
// Strip base suffixes as long as enough characters remain.
|
first = c;
|
||||||
boolean doMore = true;
|
upper = true;
|
||||||
while ( sb.length() > 3 && doMore ) {
|
|
||||||
if ( ( sb.length() + substCount > 5 ) && sb.substring( sb.length() - 2, sb.length() ).equals( "nd" ) ) {
|
|
||||||
sb.delete( sb.length() - 2, sb.length() );
|
|
||||||
}
|
|
||||||
else if ( ( sb.length() + substCount > 4 ) && sb.substring( sb.length() - 2, sb.length() ).equals( "er" ) ) {
|
|
||||||
sb.delete( sb.length() - 2, sb.length() );
|
|
||||||
}
|
|
||||||
else if ( ( sb.length() + substCount > 4 ) && sb.substring( sb.length() - 2, sb.length() ).equals( "em" ) ) {
|
|
||||||
sb.delete( sb.length() - 2, sb.length() );
|
|
||||||
}
|
|
||||||
else if ( sb.charAt( sb.length() - 1 ) == 't' ) {
|
|
||||||
sb.deleteCharAt( sb.length() - 1 );
|
|
||||||
}
|
|
||||||
else if ( sb.charAt( sb.length() - 1 ) == 'n' ) {
|
|
||||||
sb.deleteCharAt( sb.length() - 1 );
|
|
||||||
}
|
|
||||||
else if ( sb.charAt( sb.length() - 1 ) == 's' ) {
|
|
||||||
sb.deleteCharAt( sb.length() - 1 );
|
|
||||||
}
|
|
||||||
else if ( sb.charAt( sb.length() - 1 ) == 'e' ) {
|
|
||||||
sb.deleteCharAt( sb.length() - 1 );
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
doMore = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
sb = resubstitute( sb );
|
}
|
||||||
if ( !uppercase ) {
|
|
||||||
sb = removeParticleDenotion( sb );
|
|
||||||
}
|
|
||||||
return sb.toString();
|
|
||||||
}
|
}
|
||||||
|
// Discard the term if it contains a single uppercase letter that
|
||||||
/**
|
// is not starting the term.
|
||||||
* Removes a particle denotion ("ge") from a term, but only if at least 3
|
if ( first > 0 ) {
|
||||||
* characters will remain.
|
return false;
|
||||||
*
|
|
||||||
* @return The term without particle denotion, if there was one.
|
|
||||||
*/
|
|
||||||
private StringBuffer removeParticleDenotion( StringBuffer buffer ) {
|
|
||||||
for ( int c = 0; c < buffer.length(); c++ ) {
|
|
||||||
// Strip from the beginning of the string to the "ge" inclusive
|
|
||||||
if ( c < ( buffer.length() - 4 ) && buffer.charAt( c ) == 'g' && buffer.charAt ( c + 1 ) == 'e' ) {
|
|
||||||
buffer.delete( 0, c + 2 );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return sb;
|
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
/**
|
}
|
||||||
* Do some substitutions for the term to reduce overstemming:
|
/**
|
||||||
*
|
* Undoes the changes made by substitute(). That are character pairs and
|
||||||
* - Substitute Umlauts with their corresponding vowel: äöü -> aou,
|
* character combinations. Umlauts will remain as their corresponding vowel,
|
||||||
* "ß" is substituted by "ss"
|
* as "ß" remains as "ss".
|
||||||
* - Substitute a second char of an pair of equal characters with
|
*
|
||||||
* an asterisk: ?? -> ?*
|
* @return The term without the not human readable substitutions.
|
||||||
* - Substitute some common character combinations with a token:
|
*/
|
||||||
* sch/ch/ei/ie/ig/st -> $/§/%/&/#/!
|
private StringBuffer resubstitute( StringBuffer buffer ) {
|
||||||
*
|
for ( int c = 0; c < buffer.length(); c++ ) {
|
||||||
* @return The term with all needed substitutions.
|
if ( buffer.charAt( c ) == '*' ) {
|
||||||
*/
|
char x = buffer.charAt( c - 1 );
|
||||||
private StringBuffer substitute( StringBuffer buffer ) {
|
buffer.setCharAt( c, x );
|
||||||
for ( int c = 0; c < buffer.length(); c++ ) {
|
}
|
||||||
// Replace the second char of a pair of the equal characters with an asterisk
|
else if ( buffer.charAt( c ) == '$' ) {
|
||||||
if ( c > 0 && buffer.charAt( c ) == buffer.charAt ( c - 1 ) ) {
|
buffer.setCharAt( c, 's' );
|
||||||
buffer.setCharAt( c, '*' );
|
buffer.insert( c + 1, new char[]{'c', 'h'}, 0, 2 );
|
||||||
}
|
}
|
||||||
// Substitute Umlauts.
|
else if ( buffer.charAt( c ) == '§' ) {
|
||||||
else if ( buffer.charAt( c ) == 'ä' ) {
|
buffer.setCharAt( c, 'c' );
|
||||||
buffer.setCharAt( c, 'a' );
|
buffer.insert( c + 1, 'h' );
|
||||||
}
|
}
|
||||||
else if ( buffer.charAt( c ) == 'ö' ) {
|
else if ( buffer.charAt( c ) == '%' ) {
|
||||||
buffer.setCharAt( c, 'o' );
|
buffer.setCharAt( c, 'e' );
|
||||||
}
|
buffer.insert( c + 1, 'i' );
|
||||||
else if ( buffer.charAt( c ) == 'ü' ) {
|
}
|
||||||
buffer.setCharAt( c, 'u' );
|
else if ( buffer.charAt( c ) == '&' ) {
|
||||||
}
|
buffer.setCharAt( c, 'i' );
|
||||||
// Take care that at least one character is left left side from the current one
|
buffer.insert( c + 1, 'e' );
|
||||||
if ( c < buffer.length() - 1 ) {
|
}
|
||||||
if ( buffer.charAt( c ) == 'ß' ) {
|
else if ( buffer.charAt( c ) == '#' ) {
|
||||||
buffer.setCharAt( c, 's' );
|
buffer.setCharAt( c, 'i' );
|
||||||
buffer.insert( c + 1, 's' );
|
buffer.insert( c + 1, 'g' );
|
||||||
substCount++;
|
}
|
||||||
}
|
else if ( buffer.charAt( c ) == '!' ) {
|
||||||
// Masking several common character combinations with an token
|
buffer.setCharAt( c, 's' );
|
||||||
else if ( ( c < buffer.length() - 2 ) && buffer.charAt( c ) == 's' && buffer.charAt( c + 1 ) == 'c' && buffer.charAt( c + 2 ) == 'h' ) {
|
buffer.insert( c + 1, 't' );
|
||||||
buffer.setCharAt( c, '$' );
|
}
|
||||||
buffer.delete( c + 1, c + 3 );
|
|
||||||
substCount =+ 2;
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == 'c' && buffer.charAt( c + 1 ) == 'h' ) {
|
|
||||||
buffer.setCharAt( c, '§' );
|
|
||||||
buffer.deleteCharAt( c + 1 );
|
|
||||||
substCount++;
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == 'e' && buffer.charAt( c + 1 ) == 'i' ) {
|
|
||||||
buffer.setCharAt( c, '%' );
|
|
||||||
buffer.deleteCharAt( c + 1 );
|
|
||||||
substCount++;
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == 'i' && buffer.charAt( c + 1 ) == 'e' ) {
|
|
||||||
buffer.setCharAt( c, '&' );
|
|
||||||
buffer.deleteCharAt( c + 1 );
|
|
||||||
substCount++;
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == 'i' && buffer.charAt( c + 1 ) == 'g' ) {
|
|
||||||
buffer.setCharAt( c, '#' );
|
|
||||||
buffer.deleteCharAt( c + 1 );
|
|
||||||
substCount++;
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == 's' && buffer.charAt( c + 1 ) == 't' ) {
|
|
||||||
buffer.setCharAt( c, '!' );
|
|
||||||
buffer.deleteCharAt( c + 1 );
|
|
||||||
substCount++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return buffer;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks a term if it can be processed correctly.
|
|
||||||
*
|
|
||||||
* @return true if, and only if, the given term consists in letters.
|
|
||||||
*/
|
|
||||||
private boolean isStemmable( String term ) {
|
|
||||||
boolean upper = false;
|
|
||||||
int first = -1;
|
|
||||||
for ( int c = 0; c < term.length(); c++ ) {
|
|
||||||
// Discard terms that contain non-letter characters.
|
|
||||||
if ( !Character.isLetter( term.charAt( c ) ) ) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// Discard terms that contain multiple uppercase letters.
|
|
||||||
if ( Character.isUpperCase( term.charAt( c ) ) ) {
|
|
||||||
if ( upper ) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// First encountered uppercase letter, set flag and save
|
|
||||||
// position.
|
|
||||||
else {
|
|
||||||
first = c;
|
|
||||||
upper = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Discard the term if it contains a single uppercase letter that
|
|
||||||
// is not starting the term.
|
|
||||||
if ( first > 0 ) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Undoes the changes made by substitute(). That are character pairs and
|
|
||||||
* character combinations. Umlauts will remain as their corresponding vowel,
|
|
||||||
* as "ß" remains as "ss".
|
|
||||||
*
|
|
||||||
* @return The term without the not human readable substitutions.
|
|
||||||
*/
|
|
||||||
private StringBuffer resubstitute( StringBuffer buffer ) {
|
|
||||||
for ( int c = 0; c < buffer.length(); c++ ) {
|
|
||||||
if ( buffer.charAt( c ) == '*' ) {
|
|
||||||
char x = buffer.charAt( c - 1 );
|
|
||||||
buffer.setCharAt( c, x );
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == '$' ) {
|
|
||||||
buffer.setCharAt( c, 's' );
|
|
||||||
buffer.insert( c + 1, new char[]{'c', 'h'}, 0, 2 );
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == '§' ) {
|
|
||||||
buffer.setCharAt( c, 'c' );
|
|
||||||
buffer.insert( c + 1, 'h' );
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == '%' ) {
|
|
||||||
buffer.setCharAt( c, 'e' );
|
|
||||||
buffer.insert( c + 1, 'i' );
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == '&' ) {
|
|
||||||
buffer.setCharAt( c, 'i' );
|
|
||||||
buffer.insert( c + 1, 'e' );
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == '#' ) {
|
|
||||||
buffer.setCharAt( c, 'i' );
|
|
||||||
buffer.insert( c + 1, 'g' );
|
|
||||||
}
|
|
||||||
else if ( buffer.charAt( c ) == '!' ) {
|
|
||||||
buffer.setCharAt( c, 's' );
|
|
||||||
buffer.insert( c + 1, 't' );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return buffer;
|
|
||||||
}
|
}
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue