Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

- Fixed some funky indentation that I found while testing the contributed 

Portuguese stemmer.


git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@149835 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Otis Gospodnetic 2002-08-18 17:33:16 +00:00
parent bbbc192097
commit 63f7272924
4 changed files with 407 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
View File

@ -74,8 +74,8 @@ import java.util.Hashtable;
* @author Gerhard Schwarz * @author Gerhard Schwarz
* @version $Id$ * @version $Id$
*/ */
public class GermanAnalyzer extends Analyzer { public class GermanAnalyzer extends Analyzer
{
/** /**
* List of typical german stopwords. * List of typical german stopwords.
*/ */
@ -98,6 +98,7 @@ public class GermanAnalyzer extends Analyzer {
* Contains the stopwords used with the StopFilter. * Contains the stopwords used with the StopFilter.
*/ */
private Hashtable stoptable = new Hashtable(); private Hashtable stoptable = new Hashtable();
/** /**
* Contains words that should be indexed but not stemmed. * Contains words that should be indexed but not stemmed.
*/ */
@ -106,47 +107,56 @@ public class GermanAnalyzer extends Analyzer {
/** /**
* Builds an analyzer. * Builds an analyzer.
*/ */
public GermanAnalyzer() { public GermanAnalyzer()
{
stoptable = StopFilter.makeStopTable( GERMAN_STOP_WORDS ); stoptable = StopFilter.makeStopTable( GERMAN_STOP_WORDS );
} }
/** /**
* Builds an analyzer with the given stop words. * Builds an analyzer with the given stop words.
*/ */
public GermanAnalyzer( String[] stopwords ) { public GermanAnalyzer( String[] stopwords )
{
stoptable = StopFilter.makeStopTable( stopwords ); stoptable = StopFilter.makeStopTable( stopwords );
} }
/** /**
* Builds an analyzer with the given stop words. * Builds an analyzer with the given stop words.
*/ */
public GermanAnalyzer( Hashtable stopwords ) { public GermanAnalyzer( Hashtable stopwords )
{
stoptable = stopwords; stoptable = stopwords;
} }
/** /**
* Builds an analyzer with the given stop words. * Builds an analyzer with the given stop words.
*/ */
public GermanAnalyzer( File stopwords ) { public GermanAnalyzer( File stopwords )
{
stoptable = WordlistLoader.getWordtable( stopwords ); stoptable = WordlistLoader.getWordtable( stopwords );
} }
/** /**
* Builds an exclusionlist from an array of Strings. * Builds an exclusionlist from an array of Strings.
*/ */
public void setStemExclusionTable( String[] exclusionlist ) { public void setStemExclusionTable( String[] exclusionlist )
{
excltable = StopFilter.makeStopTable( exclusionlist ); excltable = StopFilter.makeStopTable( exclusionlist );
} }
/** /**
* Builds an exclusionlist from a Hashtable. * Builds an exclusionlist from a Hashtable.
*/ */
public void setStemExclusionTable( Hashtable exclusionlist ) { public void setStemExclusionTable( Hashtable exclusionlist )
{
excltable = exclusionlist; excltable = exclusionlist;
} }
/** /**
* Builds an exclusionlist from the words contained in the given file. * Builds an exclusionlist from the words contained in the given file.
*/ */
public void setStemExclusionTable( File exclusionlist ) { public void setStemExclusionTable( File exclusionlist )
{
excltable = WordlistLoader.getWordtable( exclusionlist ); excltable = WordlistLoader.getWordtable( exclusionlist );
} }
@ -156,7 +166,8 @@ public class GermanAnalyzer extends Analyzer {
* @return A TokenStream build from a StandardTokenizer filtered with * @return A TokenStream build from a StandardTokenizer filtered with
* StandardFilter, StopFilter, GermanStemFilter and LowerCaseFilter * StandardFilter, StopFilter, GermanStemFilter and LowerCaseFilter
*/ */
public TokenStream tokenStream( String fieldName, Reader reader ) { public TokenStream tokenStream( String fieldName, Reader reader )
{
TokenStream result = new StandardTokenizer( reader ); TokenStream result = new StandardTokenizer( reader );
result = new StandardFilter( result ); result = new StandardFilter( result );
result = new StopFilter( result, stoptable ); result = new StopFilter( result, stoptable );
@ -164,4 +175,3 @@ public class GermanAnalyzer extends Analyzer {
return result; return result;
} }
} }

21
src/java/org/apache/lucene/analysis/de/GermanStemFilter.java
View File

@ -68,8 +68,8 @@ import java.util.Hashtable;
* @author Gerhard Schwarz * @author Gerhard Schwarz
* @version $Id$ * @version $Id$
*/ */
public final class GermanStemFilter extends TokenFilter { public final class GermanStemFilter extends TokenFilter
{
/** /**
* The actual token in the input stream. * The actual token in the input stream.
*/ */
@ -77,7 +77,8 @@ public final class GermanStemFilter extends TokenFilter {
private GermanStemmer stemmer = null; private GermanStemmer stemmer = null;
private Hashtable exclusions = null; private Hashtable exclusions = null;
public GermanStemFilter( TokenStream in ) { public GermanStemFilter( TokenStream in )
{
stemmer = new GermanStemmer(); stemmer = new GermanStemmer();
input = in; input = in;
} }
@ -85,7 +86,8 @@ public final class GermanStemFilter extends TokenFilter {
/** /**
* Builds a GermanStemFilter that uses an exclusiontable. * Builds a GermanStemFilter that uses an exclusiontable.
*/ */
public GermanStemFilter( TokenStream in, Hashtable exclusiontable ) { public GermanStemFilter( TokenStream in, Hashtable exclusiontable )
{
this( in ); this( in );
exclusions = exclusiontable; exclusions = exclusiontable;
} }
@ -94,7 +96,8 @@ public final class GermanStemFilter extends TokenFilter {
* @return Returns the next token in the stream, or null at EOS * @return Returns the next token in the stream, or null at EOS
*/ */
public final Token next() public final Token next()
throws IOException { throws IOException
{
if ( ( token = input.next() ) == null ) { if ( ( token = input.next() ) == null ) {
return null; return null;
} }
@ -112,18 +115,22 @@ public final class GermanStemFilter extends TokenFilter {
return token; return token;
} }
} }
/** /**
* Set a alternative/custom GermanStemmer for this filter. * Set a alternative/custom GermanStemmer for this filter.
*/ */
public void setStemmer( GermanStemmer stemmer ) { public void setStemmer( GermanStemmer stemmer )
{
if ( stemmer != null ) { if ( stemmer != null ) {
this.stemmer = stemmer; this.stemmer = stemmer;
} }
} }
/** /**
* Set an alternative exclusion list for this filter. * Set an alternative exclusion list for this filter.
*/ */
public void setExclusionTable( Hashtable exclusiontable ) { public void setExclusionTable( Hashtable exclusiontable )
{
exclusions = exclusiontable; exclusions = exclusiontable;
} }
} }

45
src/java/org/apache/lucene/analysis/de/GermanStemmer.java
View File

@ -62,17 +62,18 @@ package org.apache.lucene.analysis.de;
* @author Gerhard Schwarz * @author Gerhard Schwarz
* @version $Id$ * @version $Id$
*/ */
public class GermanStemmer
public class GermanStemmer { {
/** /**
* Buffer for the terms while stemming them. * Buffer for the terms while stemming them.
*/ */
private StringBuffer sb = new StringBuffer(); private StringBuffer sb = new StringBuffer();
/** /**
* Indicates if a term is handled as a noun. * Indicates if a term is handled as a noun.
*/ */
private boolean uppercase = false; private boolean uppercase = false;
/** /**
* Amount of characters that are removed with <tt>substitute()</tt> while stemming. * Amount of characters that are removed with <tt>substitute()</tt> while stemming.
*/ */
@ -84,12 +85,14 @@ public class GermanStemmer {
* @param term The term that should be stemmed. * @param term The term that should be stemmed.
* @return Discriminator for <tt>term</tt> * @return Discriminator for <tt>term</tt>
*/ */
protected String stem( String term ) { protected String stem( String term )
{
// Mark a possible noun. // Mark a possible noun.
uppercase = Character.isUpperCase( term.charAt( 0 ) ); uppercase = Character.isUpperCase( term.charAt( 0 ) );
// Use lowercase for medium stemming. // Use lowercase for medium stemming.
term = term.toLowerCase(); term = term.toLowerCase();
if ( !isStemmable( term ) ) return term; if ( !isStemmable( term ) )
return term;
// Reset the StringBuffer. // Reset the StringBuffer.
sb.delete( 0, sb.length() ); sb.delete( 0, sb.length() );
sb.insert( 0, term ); sb.insert( 0, term );
@ -107,7 +110,8 @@ public class GermanStemmer {
* *
* @return true if, and only if, the given term consists in letters. * @return true if, and only if, the given term consists in letters.
*/ */
private boolean isStemmable( String term ) { private boolean isStemmable( String term )
{
for ( int c = 0; c < term.length(); c++ ) { for ( int c = 0; c < term.length(); c++ ) {
if ( !Character.isLetter( term.charAt( c ) ) ) return false; if ( !Character.isLetter( term.charAt( c ) ) ) return false;
} }
@ -122,16 +126,21 @@ public class GermanStemmer {
* discriminators in the most of those cases. * discriminators in the most of those cases.
* The algorithm is context free, except of the length restrictions. * The algorithm is context free, except of the length restrictions.
*/ */
private void strip( StringBuffer buffer ) { private void strip( StringBuffer buffer )
{
boolean doMore = true; boolean doMore = true;
while ( doMore && buffer.length() > 3 ) { while ( doMore && buffer.length() > 3 ) {
if ( ( buffer.length() + substCount > 5 ) && buffer.substring( buffer.length() - 2, buffer.length() ).equals( "nd" ) ) { if ( ( buffer.length() + substCount > 5 ) &&
buffer.substring( buffer.length() - 2, buffer.length() ).equals( "nd" ) )
{
buffer.delete( buffer.length() - 2, buffer.length() ); buffer.delete( buffer.length() - 2, buffer.length() );
} }
else if ( ( buffer.length() + substCount > 4 ) && buffer.substring( buffer.length() - 2, buffer.length() ).equals( "em" ) ) { else if ( ( buffer.length() + substCount > 4 ) &&
buffer.substring( buffer.length() - 2, buffer.length() ).equals( "em" ) ) {
buffer.delete( buffer.length() - 2, buffer.length() ); buffer.delete( buffer.length() - 2, buffer.length() );
} }
else if ( ( buffer.length() + substCount > 4 ) && buffer.substring( buffer.length() - 2, buffer.length() ).equals( "er" ) ) { else if ( ( buffer.length() + substCount > 4 ) &&
buffer.substring( buffer.length() - 2, buffer.length() ).equals( "er" ) ) {
buffer.delete( buffer.length() - 2, buffer.length() ); buffer.delete( buffer.length() - 2, buffer.length() );
} }
else if ( buffer.charAt( buffer.length() - 1 ) == 'e' ) { else if ( buffer.charAt( buffer.length() - 1 ) == 'e' ) {
@ -159,7 +168,8 @@ public class GermanStemmer {
* *
* @return The term with the optimizations applied. * @return The term with the optimizations applied.
*/ */
private void optimize( StringBuffer buffer ) { private void optimize( StringBuffer buffer )
{
// Additional step for female plurals of professions and inhabitants. // Additional step for female plurals of professions and inhabitants.
if ( buffer.length() > 5 && buffer.substring( buffer.length() - 5, buffer.length() ).equals( "erin*" ) ) { if ( buffer.length() > 5 && buffer.substring( buffer.length() - 5, buffer.length() ).equals( "erin*" ) ) {
buffer.deleteCharAt( buffer.length() -1 ); buffer.deleteCharAt( buffer.length() -1 );
@ -174,7 +184,8 @@ public class GermanStemmer {
/** /**
* Removes a particle denotion ("ge") from a term. * Removes a particle denotion ("ge") from a term.
*/ */
private void removeParticleDenotion( StringBuffer buffer ) { private void removeParticleDenotion( StringBuffer buffer )
{
if ( buffer.length() > 4 ) { if ( buffer.length() > 4 ) {
for ( int c = 0; c < buffer.length() - 3; c++ ) { for ( int c = 0; c < buffer.length() - 3; c++ ) {
if ( buffer.substring( c, c + 4 ).equals( "gege" ) ) { if ( buffer.substring( c, c + 4 ).equals( "gege" ) ) {
@ -195,7 +206,8 @@ public class GermanStemmer {
* - Substitute some common character combinations with a token: * - Substitute some common character combinations with a token:
* sch/ch/ei/ie/ig/st -> $/§/%/&/#/! * sch/ch/ei/ie/ig/st -> $/§/%/&/#/!
*/ */
private void substitute( StringBuffer buffer ) { private void substitute( StringBuffer buffer )
{
substCount = 0; substCount = 0;
for ( int c = 0; c < buffer.length(); c++ ) { for ( int c = 0; c < buffer.length(); c++ ) {
// Replace the second char of a pair of the equal characters with an asterisk // Replace the second char of a pair of the equal characters with an asterisk
@ -220,7 +232,9 @@ public class GermanStemmer {
substCount++; substCount++;
} }
// Masking several common character combinations with an token // Masking several common character combinations with an token
else if ( ( c < buffer.length() - 2 ) && buffer.charAt( c ) == 's' && buffer.charAt( c + 1 ) == 'c' && buffer.charAt( c + 2 ) == 'h' ) { else if ( ( c < buffer.length() - 2 ) && buffer.charAt( c ) == 's' &&
buffer.charAt( c + 1 ) == 'c' && buffer.charAt( c + 2 ) == 'h' )
{
buffer.setCharAt( c, '$' ); buffer.setCharAt( c, '$' );
buffer.delete( c + 1, c + 3 ); buffer.delete( c + 1, c + 3 );
substCount =+ 2; substCount =+ 2;
@ -259,7 +273,8 @@ public class GermanStemmer {
* character combinations. Umlauts will remain as their corresponding vowel, * character combinations. Umlauts will remain as their corresponding vowel,
* as "ß" remains as "ss". * as "ß" remains as "ss".
*/ */
private void resubstitute( StringBuffer buffer ) { private void resubstitute( StringBuffer buffer )
{
for ( int c = 0; c < buffer.length(); c++ ) { for ( int c = 0; c < buffer.length(); c++ ) {
if ( buffer.charAt( c ) == '*' ) { if ( buffer.charAt( c ) == '*' ) {
char x = buffer.charAt( c - 1 ); char x = buffer.charAt( c - 1 );

6
src/java/org/apache/lucene/analysis/de/WordlistLoader.java
View File

@ -68,8 +68,8 @@ import java.util.Hashtable;
* @author Gerhard Schwarz * @author Gerhard Schwarz
* @version $Id$ * @version $Id$
*/ */
public class WordlistLoader { public class WordlistLoader
{
/** /**
* @param path Path to the wordlist * @param path Path to the wordlist
* @param wordfile Name of the wordlist * @param wordfile Name of the wordlist
@ -80,6 +80,7 @@ public class WordlistLoader {
} }
return getWordtable( new File( path, wordfile ) ); return getWordtable( new File( path, wordfile ) );
} }
/** /**
* @param wordfile Complete path to the wordlist * @param wordfile Complete path to the wordlist
*/ */
@ -135,4 +136,3 @@ public class WordlistLoader {
return table; return table;
} }
} }