Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5859: remove dead code: changes no runtime behavior, these are all unused variables 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1614778 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-07-30 22:16:16 +00:00
parent 5b1c0f316d
commit 0368c604cc
261 changed files with 1248 additions and 1840 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
View File

@ -29,7 +29,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Arabic.
@ -89,20 +88,18 @@ public final class ArabicAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public ArabicAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public ArabicAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
*/
public ArabicAnalyzer(Version matchVersion, CharArraySet stopwords){
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public ArabicAnalyzer(CharArraySet stopwords){
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -110,17 +107,14 @@ public final class ArabicAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* {@link ArabicStemFilter}.
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
* @param stemExclusionSet
* a set of terms not to be stemmed
*/
public ArabicAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet){
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public ArabicAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet){
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -136,10 +130,10 @@ public final class ArabicAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new LowerCaseFilter(matchVersion, source);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new LowerCaseFilter(source);
// the order here is important: the stopword list is not normalized!
result = new StopFilter( matchVersion, result, stopwords);
result = new StopFilter(result, stopwords);
// TODO maybe we should make ArabicNormalization filter also KeywordAttribute aware?!
result = new ArabicNormalizationFilter(result);
if(!stemExclusionSet.isEmpty()) {

27
lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.bg;
import java.io.IOException;
import java.io.Reader;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
@ -31,7 +30,6 @@ import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Bulgarian.
@ -42,6 +40,7 @@ import org.apache.lucene.util.Version;
* <p>
*/
public final class BulgarianAnalyzer extends StopwordAnalyzerBase {
/**
* File containing default Bulgarian stopwords.
*
@ -84,15 +83,15 @@ public final class BulgarianAnalyzer extends StopwordAnalyzerBase {
* Builds an analyzer with the default stop words:
* {@link #DEFAULT_STOPWORD_FILE}.
*/
public BulgarianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public BulgarianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*/
public BulgarianAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public BulgarianAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -100,10 +99,10 @@ public final class BulgarianAnalyzer extends StopwordAnalyzerBase {
* If a stem exclusion set is provided this analyzer will add a {@link SetKeywordMarkerFilter}
* before {@link BulgarianStemFilter}.
*/
public BulgarianAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet)); }
public BulgarianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
* Creates a
@ -119,10 +118,10 @@ public final class BulgarianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
public TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new BulgarianStemFilter(result);

30
lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
View File

@ -65,7 +65,7 @@ public final class BrazilianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getWordSet(IOUtils.getDecodingReader(BrazilianAnalyzer.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), "#", Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), "#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -83,35 +83,29 @@ public final class BrazilianAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words ({@link #getDefaultStopSet()}).
*/
public BrazilianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public BrazilianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
*/
public BrazilianAnalyzer(Version matchVersion, CharArraySet stopwords) {
super(matchVersion, stopwords);
public BrazilianAnalyzer(CharArraySet stopwords) {
super(stopwords);
}
/**
* Builds an analyzer with the given stop words and stemming exclusion words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
*/
public BrazilianAnalyzer(Version matchVersion, CharArraySet stopwords,
CharArraySet stemExclusionSet) {
this(matchVersion, stopwords);
excltable = CharArraySet.unmodifiableSet(CharArraySet
.copy(matchVersion, stemExclusionSet));
public BrazilianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
this(stopwords);
excltable = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -126,10 +120,10 @@ public final class BrazilianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new LowerCaseFilter(matchVersion, source);
result = new StandardFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Tokenizer source = new StandardTokenizer();
TokenStream result = new LowerCaseFilter(source);
result = new StandardFilter(result);
result = new StopFilter(result, stopwords);
if(excltable != null && !excltable.isEmpty())
result = new SetKeywordMarkerFilter(result, excltable);
return new TokenStreamComponents(source, new BrazilianStemFilter(result));

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/ca/CatalanAnalyzer.java
View File

@ -33,7 +33,6 @@ import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.ElisionFilter;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.CatalanStemmer;
/**
@ -46,7 +45,7 @@ public final class CatalanAnalyzer extends StopwordAnalyzerBase {
public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
private static final CharArraySet DEFAULT_ARTICLES = CharArraySet.unmodifiableSet(
new CharArraySet(Version.LUCENE_CURRENT,
new CharArraySet(
Arrays.asList(
"d", "l", "m", "n", "s", "t"
), true));
@ -81,18 +80,17 @@ public final class CatalanAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public CatalanAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public CatalanAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public CatalanAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public CatalanAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -100,14 +98,12 @@ public final class CatalanAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public CatalanAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public CatalanAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -124,11 +120,11 @@ public final class CatalanAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new CatalanStemmer());

5
lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java
View File

@ -26,7 +26,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.util.Version;
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.OpenStringBuilder;
@ -29841,7 +29840,7 @@ public final class HTMLStripCharFilter extends BaseCharFilter {
upperCaseVariantsAccepted.put("amp", "AMP");
}
private static final CharArrayMap<Character> entityValues
= new CharArrayMap<>(Version.LUCENE_CURRENT, 253, false);
= new CharArrayMap<>(253, false);
static {
String[] entities = {
"AElig", "\u00C6", "Aacute", "\u00C1", "Acirc", "\u00C2",
@ -29980,7 +29979,7 @@ public final class HTMLStripCharFilter extends BaseCharFilter {
escapeSTYLE = true;
} else {
if (null == this.escapedTags) {
this.escapedTags = new CharArraySet(Version.LUCENE_CURRENT, 16, true);
this.escapedTags = new CharArraySet(16, true);
}
this.escapedTags.add(tag);
}

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
View File

@ -24,7 +24,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.util.Version;
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.OpenStringBuilder;
@ -195,7 +194,7 @@ InlineElment = ( [aAbBiIqQsSuU] |
escapeSTYLE = true;
} else {
if (null == this.escapedTags) {
this.escapedTags = new CharArraySet(Version.LUCENE_CURRENT, 16, true);
this.escapedTags = new CharArraySet(16, true);
}
this.escapedTags.add(tag);
}

19
lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.analysis.cjk;
*/
import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@ -28,7 +27,6 @@ import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
/**
* An {@link Analyzer} that tokenizes text with {@link StandardTokenizer},
@ -37,6 +35,7 @@ import org.apache.lucene.util.Version;
* and filters stopwords with {@link StopFilter}
*/
public final class CJKAnalyzer extends StopwordAnalyzerBase {
/**
* File containing default CJK stopwords.
* <p/>
@ -70,29 +69,27 @@ public final class CJKAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer which removes words in {@link #getDefaultStopSet()}.
*/
public CJKAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public CJKAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
*/
public CJKAnalyzer(Version matchVersion, CharArraySet stopwords){
super(matchVersion, stopwords);
public CJKAnalyzer(CharArraySet stopwords){
super(stopwords);
}
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
final Tokenizer source = new StandardTokenizer();
// run the widthfilter first before bigramming, it sometimes combines characters.
TokenStream result = new CJKWidthFilter(source);
result = new LowerCaseFilter(matchVersion, result);
result = new LowerCaseFilter(result);
result = new CJKBigramFilter(result);
return new TokenStreamComponents(source, new StopFilter(matchVersion, result, stopwords));
return new TokenStreamComponents(source, new StopFilter(result, stopwords));
}
}

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniAnalyzer.java
View File

@ -33,7 +33,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Sorani Kurdish.
@ -62,7 +61,7 @@ public final class SoraniAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getWordSet(IOUtils.getDecodingReader(SoraniAnalyzer.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -74,18 +73,17 @@ public final class SoraniAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public SoraniAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public SoraniAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public SoraniAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public SoraniAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -93,14 +91,12 @@ public final class SoraniAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public SoraniAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public SoraniAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -118,11 +114,11 @@ public final class SoraniAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new SoraniNormalizationFilter(result);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SoraniStemFilter(result);

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
View File

@ -78,7 +78,7 @@ public final class CommonGramsFilter extends TokenFilter {
* @param input TokenStream input in filter chain
* @param commonWords The set of common words.
*/
public CommonGramsFilter(Version matchVersion, TokenStream input, CharArraySet commonWords) {
public CommonGramsFilter(TokenStream input, CharArraySet commonWords) {
super(input);
this.commonWords = commonWords;
}

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilterFactory.java
View File

@ -76,7 +76,7 @@ public class CommonGramsFilterFactory extends TokenFilterFactory implements Reso
@Override
public TokenFilter create(TokenStream input) {
CommonGramsFilter commonGrams = new CommonGramsFilter(luceneMatchVersion, input, commonWords);
CommonGramsFilter commonGrams = new CommonGramsFilter(input, commonWords);
return commonGrams;
}
}

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordAnalyzer.java
View File

@ -17,8 +17,6 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
/**

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java
View File

@ -18,13 +18,11 @@ package org.apache.lucene.analysis.core;
*/
import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource;
/**
* Emits the entire input as a single token.

1
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.analysis.core;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.AttributeFactory;
import java.io.Reader;
import java.util.Map;
/**

23
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizer.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.analysis.core;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.Version;
/**
* A LetterTokenizer is a tokenizer that divides text at non-letters. That's to
@ -30,41 +29,25 @@ import org.apache.lucene.util.Version;
* Note: this does a decent job for most European languages, but does a terrible
* job for some Asian languages, where words are not separated by spaces.
* </p>
* <p>
* <a name="version"/>
* You must specify the required {@link Version} compatibility when creating
* {@link LetterTokenizer}:
* <ul>
* <li>As of 3.1, {@link CharTokenizer} uses an int based API to normalize and
* detect token characters. See {@link CharTokenizer#isTokenChar(int)} and
* {@link CharTokenizer#normalize(int)} for details.</li>
* </ul>
* </p>
*/
public class LetterTokenizer extends CharTokenizer {
/**
* Construct a new LetterTokenizer.
*
* @param matchVersion
* Lucene version to match See {@link <a href="#version">above</a>}
*/
public LetterTokenizer(Version matchVersion) {
super(matchVersion);
public LetterTokenizer() {
}
/**
* Construct a new LetterTokenizer using a given
* {@link org.apache.lucene.util.AttributeFactory}.
*
* @param matchVersion
* Lucene version to match See {@link <a href="#version">above</a>}
* @param factory
* the attribute factory to use for this {@link Tokenizer}
*/
public LetterTokenizer(Version matchVersion, AttributeFactory factory) {
super(matchVersion, factory);
public LetterTokenizer(AttributeFactory factory) {
super(factory);
}
/** Collects only characters which satisfy

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java
View File

@ -36,7 +36,6 @@ public class LetterTokenizerFactory extends TokenizerFactory {
/** Creates a new LetterTokenizerFactory */
public LetterTokenizerFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@ -44,6 +43,6 @@ public class LetterTokenizerFactory extends TokenizerFactory {
@Override
public LetterTokenizer create(AttributeFactory factory) {
return new LetterTokenizer(luceneMatchVersion, factory);
return new LetterTokenizer(factory);
}
}

13
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseFilter.java
View File

@ -23,30 +23,21 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharacterUtils;
import org.apache.lucene.util.Version;
/**
* Normalizes token text to lower case.
* <a name="version"/>
* <p>You must specify the required {@link Version}
* compatibility when creating LowerCaseFilter:
* <ul>
* <li> As of 3.1, supplementary characters are properly lowercased.
* </ul>
*/
public final class LowerCaseFilter extends TokenFilter {
private final CharacterUtils charUtils;
private final CharacterUtils charUtils = CharacterUtils.getInstance();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
/**
* Create a new LowerCaseFilter, that normalizes token text to lower case.
*
* @param matchVersion See <a href="#version">above</a>
* @param in TokenStream to filter
*/
public LowerCaseFilter(Version matchVersion, TokenStream in) {
public LowerCaseFilter(TokenStream in) {
super(in);
charUtils = CharacterUtils.getInstance(matchVersion);
}
@Override

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseFilterFactory.java
View File

@ -40,7 +40,6 @@ public class LowerCaseFilterFactory extends TokenFilterFactory implements MultiT
/** Creates a new LowerCaseFilterFactory */
public LowerCaseFilterFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@ -48,7 +47,7 @@ public class LowerCaseFilterFactory extends TokenFilterFactory implements MultiT
@Override
public LowerCaseFilter create(TokenStream input) {
return new LowerCaseFilter(luceneMatchVersion,input);
return new LowerCaseFilter(input);
}
@Override

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java
View File

@ -17,13 +17,8 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
* LowerCaseTokenizer performs the function of LetterTokenizer
@ -35,41 +30,24 @@ import org.apache.lucene.util.Version;
* Note: this does a decent job for most European languages, but does a terrible
* job for some Asian languages, where words are not separated by spaces.
* </p>
* <p>
* <a name="version"/>
* You must specify the required {@link Version} compatibility when creating
* {@link LowerCaseTokenizer}:
* <ul>
* <li>As of 3.1, {@link CharTokenizer} uses an int based API to normalize and
* detect token characters. See {@link CharTokenizer#isTokenChar(int)} and
* {@link CharTokenizer#normalize(int)} for details.</li>
* </ul>
* </p>
*/
public final class LowerCaseTokenizer extends LetterTokenizer {
/**
* Construct a new LowerCaseTokenizer.
*
* @param matchVersion
* Lucene version to match See {@link <a href="#version">above</a>}
*
*/
public LowerCaseTokenizer(Version matchVersion) {
super(matchVersion);
public LowerCaseTokenizer() {
}
/**
* Construct a new LowerCaseTokenizer using a given
* {@link org.apache.lucene.util.AttributeFactory}.
*
* @param matchVersion
* Lucene version to match See {@link <a href="#version">above</a>}
* @param factory
* the attribute factory to use for this {@link Tokenizer}
*/
public LowerCaseTokenizer(Version matchVersion, AttributeFactory factory) {
super(matchVersion, factory);
public LowerCaseTokenizer(AttributeFactory factory) {
super(factory);
}
/** Converts char to lower case

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java
View File

@ -39,7 +39,6 @@ public class LowerCaseTokenizerFactory extends TokenizerFactory implements Multi
/** Creates a new LowerCaseTokenizerFactory */
public LowerCaseTokenizerFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@ -47,7 +46,7 @@ public class LowerCaseTokenizerFactory extends TokenizerFactory implements Multi
@Override
public LowerCaseTokenizer create(AttributeFactory factory) {
return new LowerCaseTokenizer(luceneMatchVersion, factory);
return new LowerCaseTokenizer(factory);
}
@Override

20
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/SimpleAnalyzer.java
View File

@ -17,38 +17,22 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.util.Version;
/** An {@link Analyzer} that filters {@link LetterTokenizer}
* with {@link LowerCaseFilter}
* <p>
* <a name="version">You must specify the required {@link Version} compatibility
* when creating {@link CharTokenizer}:
* <ul>
* <li>As of 3.1, {@link LowerCaseTokenizer} uses an int based API to normalize and
* detect token codepoints. See {@link CharTokenizer#isTokenChar(int)} and
* {@link CharTokenizer#normalize(int)} for details.</li>
* </ul>
* <p>
**/
public final class SimpleAnalyzer extends Analyzer {
private final Version matchVersion;
/**
* Creates a new {@link SimpleAnalyzer}
* @param matchVersion Lucene version to match See {@link <a href="#version">above</a>}
*/
public SimpleAnalyzer(Version matchVersion) {
this.matchVersion = matchVersion;
public SimpleAnalyzer() {
}
@Override
protected TokenStreamComponents createComponents(final String fieldName) {
return new TokenStreamComponents(new LowerCaseTokenizer(matchVersion));
return new TokenStreamComponents(new LowerCaseTokenizer());
}
}

48
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopAnalyzer.java
View File

@ -27,20 +27,10 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.Version;
/** Filters {@link LetterTokenizer} with {@link LowerCaseFilter} and {@link StopFilter}.
*
* <a name="version"/>
* <p>You must specify the required {@link Version}
* compatibility when creating StopAnalyzer:
* <ul>
* <li> As of 3.1, StopFilter correctly handles Unicode 4.0
* supplementary characters in stopwords
* <li> As of 2.9, position increments are preserved
* </ul>
*/
/**
* Filters {@link LetterTokenizer} with {@link LowerCaseFilter} and {@link StopFilter}.
*/
public final class StopAnalyzer extends StopwordAnalyzerBase {
/** An unmodifiable set containing some common English words that are not usually useful
@ -55,40 +45,35 @@ public final class StopAnalyzer extends StopwordAnalyzerBase {
"that", "the", "their", "then", "there", "these",
"they", "this", "to", "was", "will", "with"
);
final CharArraySet stopSet = new CharArraySet(Version.LUCENE_CURRENT,
stopWords, false);
final CharArraySet stopSet = new CharArraySet(stopWords, false);
ENGLISH_STOP_WORDS_SET = CharArraySet.unmodifiableSet(stopSet);
}
/** Builds an analyzer which removes words in
* {@link #ENGLISH_STOP_WORDS_SET}.
* @param matchVersion See <a href="#version">above</a>
*/
public StopAnalyzer(Version matchVersion) {
this(matchVersion, ENGLISH_STOP_WORDS_SET);
public StopAnalyzer() {
this(ENGLISH_STOP_WORDS_SET);
}
/** Builds an analyzer with the stop words from the given set.
* @param matchVersion See <a href="#version">above</a>
* @param stopWords Set of stop words */
public StopAnalyzer(Version matchVersion, CharArraySet stopWords) {
super(matchVersion, stopWords);
public StopAnalyzer(CharArraySet stopWords) {
super(stopWords);
}
/** Builds an analyzer with the stop words from the given file.
* @see WordlistLoader#getWordSet(Reader, Version)
* @param matchVersion See <a href="#version">above</a>
* @see WordlistLoader#getWordSet(Reader)
* @param stopwordsFile File to load stop words from */
public StopAnalyzer(Version matchVersion, File stopwordsFile) throws IOException {
this(matchVersion, loadStopwordSet(stopwordsFile, matchVersion));
public StopAnalyzer(File stopwordsFile) throws IOException {
this(loadStopwordSet(stopwordsFile));
}
/** Builds an analyzer with the stop words from the given reader.
* @see WordlistLoader#getWordSet(Reader, Version)
* @param matchVersion See <a href="#version">above</a>
* @see WordlistLoader#getWordSet(Reader)
* @param stopwords Reader to load stop words from */
public StopAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
this(matchVersion, loadStopwordSet(stopwords, matchVersion));
public StopAnalyzer(Reader stopwords) throws IOException {
this(loadStopwordSet(stopwords));
}
/**
@ -102,9 +87,8 @@ public final class StopAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new LowerCaseTokenizer(matchVersion);
return new TokenStreamComponents(source, new StopFilter(matchVersion,
source, stopwords));
final Tokenizer source = new LowerCaseTokenizer();
return new TokenStreamComponents(source, new StopFilter(source, stopwords));
}
}

43
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java
View File

@ -24,19 +24,9 @@ import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
/**
* Removes stop words from a token stream.
*
* <a name="version"/>
* <p>You must specify the required {@link Version}
* compatibility when creating StopFilter:
* <ul>
* <li> As of 3.1, StopFilter correctly handles Unicode 4.0
* supplementary characters in stopwords and position
* increments are preserved
* </ul>
*/
public final class StopFilter extends FilteringTokenFilter {
@ -47,17 +37,14 @@ public final class StopFilter extends FilteringTokenFilter {
* Constructs a filter which removes words from the input TokenStream that are
* named in the Set.
*
* @param matchVersion
* Lucene version to enable correct Unicode 4.0 behavior in the stop
* set if Version > 3.0. See <a href="#version">above</a> for details.
* @param in
* Input stream
* @param stopWords
* A {@link CharArraySet} representing the stopwords.
* @see #makeStopSet(Version, java.lang.String...)
* @see #makeStopSet(java.lang.String...)
*/
public StopFilter(Version matchVersion, TokenStream in, CharArraySet stopWords) {
super(matchVersion, in);
public StopFilter(TokenStream in, CharArraySet stopWords) {
super(in);
this.stopWords = stopWords;
}
@ -67,12 +54,11 @@ public final class StopFilter extends FilteringTokenFilter {
* This permits this stopWords construction to be cached once when
* an Analyzer is constructed.
*
* @param matchVersion Lucene version to enable correct Unicode 4.0 behavior in the returned set if Version > 3.0
* @param stopWords An array of stopwords
* @see #makeStopSet(Version, java.lang.String[], boolean) passing false to ignoreCase
* @see #makeStopSet(java.lang.String[], boolean) passing false to ignoreCase
*/
public static CharArraySet makeStopSet(Version matchVersion, String... stopWords) {
return makeStopSet(matchVersion, stopWords, false);
public static CharArraySet makeStopSet(String... stopWords) {
return makeStopSet(stopWords, false);
}
/**
@ -81,38 +67,35 @@ public final class StopFilter extends FilteringTokenFilter {
* This permits this stopWords construction to be cached once when
* an Analyzer is constructed.
*
* @param matchVersion Lucene version to enable correct Unicode 4.0 behavior in the returned set if Version > 3.0
* @param stopWords A List of Strings or char[] or any other toString()-able list representing the stopwords
* @return A Set ({@link CharArraySet}) containing the words
* @see #makeStopSet(Version, java.lang.String[], boolean) passing false to ignoreCase
* @see #makeStopSet(java.lang.String[], boolean) passing false to ignoreCase
*/
public static CharArraySet makeStopSet(Version matchVersion, List<?> stopWords) {
return makeStopSet(matchVersion, stopWords, false);
public static CharArraySet makeStopSet(List<?> stopWords) {
return makeStopSet(stopWords, false);
}
/**
* Creates a stopword set from the given stopword array.
*
* @param matchVersion Lucene version to enable correct Unicode 4.0 behavior in the returned set if Version > 3.0
* @param stopWords An array of stopwords
* @param ignoreCase If true, all words are lower cased first.
* @return a Set containing the words
*/
public static CharArraySet makeStopSet(Version matchVersion, String[] stopWords, boolean ignoreCase) {
CharArraySet stopSet = new CharArraySet(matchVersion, stopWords.length, ignoreCase);
public static CharArraySet makeStopSet(String[] stopWords, boolean ignoreCase) {
CharArraySet stopSet = new CharArraySet(stopWords.length, ignoreCase);
stopSet.addAll(Arrays.asList(stopWords));
return stopSet;
}
/**
* Creates a stopword set from the given stopword list.
* @param matchVersion Lucene version to enable correct Unicode 4.0 behavior in the returned set if Version > 3.0
* @param stopWords A List of Strings or char[] or any other toString()-able list representing the stopwords
* @param ignoreCase if true, all words are lower cased first
* @return A Set ({@link CharArraySet}) containing the words
*/
public static CharArraySet makeStopSet(Version matchVersion, List<?> stopWords, boolean ignoreCase){
CharArraySet stopSet = new CharArraySet(matchVersion, stopWords.size(), ignoreCase);
public static CharArraySet makeStopSet(List<?> stopWords, boolean ignoreCase){
CharArraySet stopSet = new CharArraySet(stopWords.size(), ignoreCase);
stopSet.addAll(stopWords);
return stopSet;
}

5
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java
View File

@ -81,7 +81,6 @@ public class StopFilterFactory extends TokenFilterFactory implements ResourceLoa
/** Creates a new StopFilterFactory */
public StopFilterFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
stopWordFiles = get(args, "words");
format = get(args, "format", (null == stopWordFiles ? null : FORMAT_WORDSET));
ignoreCase = getBoolean(args, "ignoreCase", false);
@ -104,7 +103,7 @@ public class StopFilterFactory extends TokenFilterFactory implements ResourceLoa
if (null != format) {
throw new IllegalArgumentException("'format' can not be specified w/o an explicit 'words' file: " + format);
}
stopWords = new CharArraySet(luceneMatchVersion, StopAnalyzer.ENGLISH_STOP_WORDS_SET, ignoreCase);
stopWords = new CharArraySet(StopAnalyzer.ENGLISH_STOP_WORDS_SET, ignoreCase);
}
}
@ -118,7 +117,7 @@ public class StopFilterFactory extends TokenFilterFactory implements ResourceLoa
@Override
public TokenStream create(TokenStream input) {
StopFilter stopFilter = new StopFilter(luceneMatchVersion,input,stopWords);
StopFilter stopFilter = new StopFilter(input,stopWords);
return stopFilter;
}
}

11
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java
View File

@ -22,7 +22,6 @@ import java.util.Set;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.util.Version;
/**
* Removes tokens whose types appear in a set of blocked types from a token stream.
@ -35,14 +34,13 @@ public final class TypeTokenFilter extends FilteringTokenFilter {
/**
* Create a new {@link TypeTokenFilter}.
* @param version the Lucene match version
* @param input the {@link TokenStream} to consume
* @param stopTypes the types to filter
* @param useWhiteList if true, then tokens whose type is in stopTypes will
* be kept, otherwise they will be filtered out
*/
public TypeTokenFilter(Version version, TokenStream input, Set<String> stopTypes, boolean useWhiteList) {
super(version, input);
public TypeTokenFilter(TokenStream input, Set<String> stopTypes, boolean useWhiteList) {
super(input);
this.stopTypes = stopTypes;
this.useWhiteList = useWhiteList;
}
@ -50,10 +48,9 @@ public final class TypeTokenFilter extends FilteringTokenFilter {
/**
* Create a new {@link TypeTokenFilter} that filters tokens out
* (useWhiteList=false).
* @see #TypeTokenFilter(Version, TokenStream, Set, boolean)
*/
public TypeTokenFilter(Version version, TokenStream input, Set<String> stopTypes) {
this(version, input, stopTypes, false);
public TypeTokenFilter(TokenStream input, Set<String> stopTypes) {
this(input, stopTypes, false);
}
/**

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java
View File

@ -72,7 +72,7 @@ public class TypeTokenFilterFactory extends TokenFilterFactory implements Resour
@Override
public TokenStream create(TokenStream input) {
final TokenStream filter = new TypeTokenFilter(luceneMatchVersion, input, stopTypes, useWhitelist);
final TokenStream filter = new TypeTokenFilter(input, stopTypes, useWhitelist);
return filter;
}
}

10
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UpperCaseFilter.java
View File

@ -23,13 +23,9 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharacterUtils;
import org.apache.lucene.util.Version;
/**
* Normalizes token text to UPPER CASE.
* <a name="version"/>
* <p>You must specify the required {@link Version}
* compatibility when creating UpperCaseFilter
*
* <p><b>NOTE:</b> In Unicode, this transformation may lose information when the
* upper case character represents more than one lower case character. Use this filter
@ -37,18 +33,16 @@ import org.apache.lucene.util.Version;
* general search matching
*/
public final class UpperCaseFilter extends TokenFilter {
private final CharacterUtils charUtils;
private final CharacterUtils charUtils = CharacterUtils.getInstance();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
/**
* Create a new UpperCaseFilter, that normalizes token text to upper case.
*
* @param matchVersion See <a href="#version">above</a>
* @param in TokenStream to filter
*/
public UpperCaseFilter(Version matchVersion, TokenStream in) {
public UpperCaseFilter(TokenStream in) {
super(in);
charUtils = CharacterUtils.getInstance(matchVersion);
}
@Override

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UpperCaseFilterFactory.java
View File

@ -45,7 +45,6 @@ public class UpperCaseFilterFactory extends TokenFilterFactory implements MultiT
/** Creates a new UpperCaseFilterFactory */
public UpperCaseFilterFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@ -53,7 +52,7 @@ public class UpperCaseFilterFactory extends TokenFilterFactory implements MultiT
@Override
public UpperCaseFilter create(TokenStream input) {
return new UpperCaseFilter(luceneMatchVersion,input);
return new UpperCaseFilter(input);
}
@Override

21
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceAnalyzer.java
View File

@ -17,38 +17,21 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.util.Version;
/**
* An Analyzer that uses {@link WhitespaceTokenizer}.
* <p>
* <a name="version">You must specify the required {@link Version} compatibility
* when creating {@link CharTokenizer}:
* <ul>
* <li>As of 3.1, {@link WhitespaceTokenizer} uses an int based API to normalize and
* detect token codepoints. See {@link CharTokenizer#isTokenChar(int)} and
* {@link CharTokenizer#normalize(int)} for details.</li>
* </ul>
* <p>
**/
public final class WhitespaceAnalyzer extends Analyzer {
private final Version matchVersion;
/**
* Creates a new {@link WhitespaceAnalyzer}
* @param matchVersion Lucene version to match See {@link <a href="#version">above</a>}
*/
public WhitespaceAnalyzer(Version matchVersion) {
this.matchVersion = matchVersion;
public WhitespaceAnalyzer() {
}
@Override
protected TokenStreamComponents createComponents(final String fieldName) {
return new TokenStreamComponents(new WhitespaceTokenizer(matchVersion));
return new TokenStreamComponents(new WhitespaceTokenizer());
}
}

29
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java
View File

@ -17,50 +17,31 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
* A WhitespaceTokenizer is a tokenizer that divides text at whitespace.
* Adjacent sequences of non-Whitespace characters form tokens. <a
* name="version"/>
* <p>
* You must specify the required {@link Version} compatibility when creating
* {@link WhitespaceTokenizer}:
* <ul>
* <li>As of 3.1, {@link CharTokenizer} uses an int based API to normalize and
* detect token characters. See {@link CharTokenizer#isTokenChar(int)} and
* {@link CharTokenizer#normalize(int)} for details.</li>
* </ul>
* Adjacent sequences of non-Whitespace characters form tokens.
*/
public final class WhitespaceTokenizer extends CharTokenizer {
/**
* Construct a new WhitespaceTokenizer. * @param matchVersion Lucene version
* to match See {@link <a href="#version">above</a>}
*
* Construct a new WhitespaceTokenizer.
*/
public WhitespaceTokenizer(Version matchVersion) {
super(matchVersion);
public WhitespaceTokenizer() {
}
/**
* Construct a new WhitespaceTokenizer using a given
* {@link org.apache.lucene.util.AttributeFactory}.
*
* @param
* matchVersion Lucene version to match See
* {@link <a href="#version">above</a>}
* @param factory
* the attribute factory to use for this {@link Tokenizer}
*/
public WhitespaceTokenizer(Version matchVersion, AttributeFactory factory) {
super(matchVersion, factory);
public WhitespaceTokenizer(AttributeFactory factory) {
super(factory);
}
/** Collects only characters which do not satisfy

4
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.analysis.core;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.AttributeFactory;
import java.io.Reader;
import java.util.Map;
/**
@ -37,7 +36,6 @@ public class WhitespaceTokenizerFactory extends TokenizerFactory {
/** Creates a new WhitespaceTokenizerFactory */
public WhitespaceTokenizerFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@ -45,6 +43,6 @@ public class WhitespaceTokenizerFactory extends TokenizerFactory {
@Override
public WhitespaceTokenizer create(AttributeFactory factory) {
return new WhitespaceTokenizer(luceneMatchVersion, factory);
return new WhitespaceTokenizer(factory);
}
}

31
lucene/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
View File

@ -29,7 +29,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import java.io.*;
import java.nio.charset.StandardCharsets;
@ -61,7 +60,7 @@ public final class CzechAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_SET = WordlistLoader.getWordSet(IOUtils.getDecodingReader(CzechAnalyzer.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), "#", Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), "#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -75,34 +74,30 @@ public final class CzechAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words ({@link #getDefaultStopSet()}).
*
* @param matchVersion Lucene version to match
*/
public CzechAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_SET);
public CzechAnalyzer() {
this(DefaultSetHolder.DEFAULT_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion Lucene version to match
* @param stopwords a stopword set
*/
public CzechAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public CzechAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
* Builds an analyzer with the given stop words and a set of work to be
* excluded from the {@link CzechStemFilter}.
*
* @param matchVersion Lucene version to match
* @param stopwords a stopword set
* @param stemExclusionTable a stemming exclusion set
*/
public CzechAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionTable) {
super(matchVersion, stopwords);
this.stemExclusionTable = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion, stemExclusionTable));
public CzechAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionTable) {
super(stopwords);
this.stemExclusionTable = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionTable));
}
/**
@ -115,16 +110,16 @@ public final class CzechAnalyzer extends StopwordAnalyzerBase {
* {@link StandardFilter}, {@link LowerCaseFilter}, {@link StopFilter}
* , and {@link CzechStemFilter} (only if version is >= LUCENE_31). If
* a stem exclusion set is provided via
* {@link #CzechAnalyzer(Version, CharArraySet, CharArraySet)} a
* {@link #CzechAnalyzer(CharArraySet, CharArraySet)} a
* {@link SetKeywordMarkerFilter} is added before
* {@link CzechStemFilter}.
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter( matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!this.stemExclusionTable.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionTable);
result = new CzechStemFilter(result);

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/da/DanishAnalyzer.java
View File

@ -34,7 +34,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.DanishStemmer;
/**
@ -64,7 +63,7 @@ public final class DanishAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -76,18 +75,17 @@ public final class DanishAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public DanishAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public DanishAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public DanishAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public DanishAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -95,14 +93,12 @@ public final class DanishAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public DanishAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public DanishAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -119,10 +115,10 @@ public final class DanishAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new DanishStemmer());

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
View File

@ -69,7 +69,7 @@ public final class GermanAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -91,35 +91,31 @@ public final class GermanAnalyzer extends StopwordAnalyzerBase {
* Builds an analyzer with the default stop words:
* {@link #getDefaultStopSet()}.
*/
public GermanAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_SET);
public GermanAnalyzer() {
this(DefaultSetHolder.DEFAULT_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
*/
public GermanAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public GermanAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
* @param stemExclusionSet
* a stemming exclusion set
*/
public GermanAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
exclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion, stemExclusionSet));
public GermanAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
exclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -135,10 +131,10 @@ public final class GermanAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter( matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
result = new SetKeywordMarkerFilter(result, exclusionSet);
result = new GermanNormalizationFilter(result);
result = new GermanLightStemFilter(result);

18
lucene/analysis/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java
View File

@ -69,10 +69,9 @@ public final class GreekAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words.
* @param matchVersion Lucene compatibility version
*/
public GreekAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_SET);
public GreekAnalyzer() {
this(DefaultSetHolder.DEFAULT_SET);
}
/**
@ -81,11 +80,10 @@ public final class GreekAnalyzer extends StopwordAnalyzerBase {
* <b>NOTE:</b> The stopwords set should be pre-processed with the logic of
* {@link GreekLowerCaseFilter} for best results.
*
* @param matchVersion Lucene compatibility version
* @param stopwords a stopword set
*/
public GreekAnalyzer(Version matchVersion, CharArraySet stopwords) {
super(matchVersion, stopwords);
public GreekAnalyzer(CharArraySet stopwords) {
super(stopwords);
}
/**
@ -100,10 +98,10 @@ public final class GreekAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new GreekLowerCaseFilter(matchVersion, source);
result = new StandardFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new GreekLowerCaseFilter(source);
result = new StandardFilter(result);
result = new StopFilter(result, stopwords);
result = new GreekStemFilter(result);
return new TokenStreamComponents(source, result);
}

14
lucene/analysis/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java
View File

@ -22,32 +22,22 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharacterUtils;
import org.apache.lucene.util.Version;
/**
* Normalizes token text to lower case, removes some Greek diacritics,
* and standardizes final sigma to sigma.
* <a name="version"/>
* <p>You must specify the required {@link Version}
* compatibility when creating GreekLowerCaseFilter:
* <ul>
* <li> As of 3.1, supplementary characters are properly lowercased.
* </ul>
*/
public final class GreekLowerCaseFilter extends TokenFilter {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final CharacterUtils charUtils;
private final CharacterUtils charUtils = CharacterUtils.getInstance();
/**
* Create a GreekLowerCaseFilter that normalizes Greek token text.
*
* @param matchVersion Lucene compatibility version,
* See <a href="#version">above</a>
* @param in TokenStream to filter
*/
public GreekLowerCaseFilter(Version matchVersion, TokenStream in) {
public GreekLowerCaseFilter(TokenStream in) {
super(in);
this.charUtils = CharacterUtils.getInstance(matchVersion);
}
@Override

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilterFactory.java
View File

@ -40,7 +40,6 @@ public class GreekLowerCaseFilterFactory extends TokenFilterFactory implements M
/** Creates a new GreekLowerCaseFilterFactory */
public GreekLowerCaseFilterFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@ -48,7 +47,7 @@ public class GreekLowerCaseFilterFactory extends TokenFilterFactory implements M
@Override
public GreekLowerCaseFilter create(TokenStream in) {
return new GreekLowerCaseFilter(luceneMatchVersion, in);
return new GreekLowerCaseFilter(in);
}
@Override

33
lucene/analysis/common/src/java/org/apache/lucene/analysis/el/GreekStemmer.java
View File

@ -1,7 +1,6 @@
package org.apache.lucene.analysis.el;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
import java.util.Arrays;
@ -205,7 +204,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc4 = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc4 = new CharArraySet(
Arrays.asList("θ", "δ", "ελ", "γαλ", "ν", "π", "ιδ", "παρ"),
false);
@ -231,7 +230,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc6 = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc6 = new CharArraySet(
Arrays.asList("αλ", "αδ", "ενδ", "αμαν", "αμμοχαλ", "ηθ", "ανηθ",
"αντιδ", "φυσ", "βρωμ", "γερ", "εξωδ", "καλπ", "καλλιν", "καταδ",
"μουλ", "μπαν", "μπαγιατ", "μπολ", "μποσ", "νιτ", "ξικ", "συνομηλ",
@ -256,7 +255,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc7 = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc7 = new CharArraySet(
Arrays.asList("αναπ", "αποθ", "αποκ", "αποστ", "βουβ", "ξεθ", "ουλ",
"πεθ", "πικρ", "ποτ", "σιχ", "χ"),
false);
@ -283,11 +282,11 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc8a = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc8a = new CharArraySet(
Arrays.asList("τρ", "τσ"),
false);
private static final CharArraySet exc8b = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc8b = new CharArraySet(
Arrays.asList("βετερ", "βουλκ", "βραχμ", "γ", "δραδουμ", "θ", "καλπουζ",
"καστελ", "κορμορ", "λαοπλ", "μωαμεθ", "μ", "μουσουλμ", "ν", "ουλ",
"π", "πελεκ", "πλ", "πολισ", "πορτολ", "σαρακατσ", "σουλτ",
@ -346,7 +345,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc9 = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc9 = new CharArraySet(
Arrays.asList("αβαρ", "βεν", "εναρ", "αβρ", "αδ", "αθ", "αν", "απλ",
"βαρον", "ντρ", "σκ", "κοπ", "μπορ", "νιφ", "παγ", "παρακαλ", "σερπ",
"σκελ", "συρφ", "τοκ", "υ", "δ", "εμ", "θαρρ", "θ"),
@ -434,11 +433,11 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc12a = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc12a = new CharArraySet(
Arrays.asList("π", "απ", "συμπ", "ασυμπ", "ακαταπ", "αμεταμφ"),
false);
private static final CharArraySet exc12b = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc12b = new CharArraySet(
Arrays.asList("αλ", "αρ", "εκτελ", "ζ", "μ", "ξ", "παρακαλ", "αρ", "προ", "νισ"),
false);
@ -458,7 +457,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc13 = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc13 = new CharArraySet(
Arrays.asList("διαθ", "θ", "παρακαταθ", "προσθ", "συνθ"),
false);
@ -492,7 +491,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc14 = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc14 = new CharArraySet(
Arrays.asList("φαρμακ", "χαδ", "αγκ", "αναρρ", "βρομ", "εκλιπ", "λαμπιδ",
"λεχ", "μ", "πατ", "ρ", "λ", "μεδ", "μεσαζ", "υποτειν", "αμ", "αιθ",
"ανηκ", "δεσποζ", "ενδιαφερ", "δε", "δευτερευ", "καθαρευ", "πλε",
@ -530,7 +529,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc15a = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc15a = new CharArraySet(
Arrays.asList("αβαστ", "πολυφ", "αδηφ", "παμφ", "ρ", "ασπ", "αφ", "αμαλ",
"αμαλλι", "ανυστ", "απερ", "ασπαρ", "αχαρ", "δερβεν", "δροσοπ",
"ξεφ", "νεοπ", "νομοτ", "ολοπ", "ομοτ", "προστ", "προσωποπ", "συμπ",
@ -539,7 +538,7 @@ public class GreekStemmer {
"ουλαμ", "ουρ", "π", "τρ", "μ"),
false);
private static final CharArraySet exc15b = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc15b = new CharArraySet(
Arrays.asList("ψοφ", "ναυλοχ"),
false);
@ -576,7 +575,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc16 = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc16 = new CharArraySet(
Arrays.asList("ν", "χερσον", "δωδεκαν", "ερημον", "μεγαλον", "επταν"),
false);
@ -596,7 +595,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc17 = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc17 = new CharArraySet(
Arrays.asList("ασβ", "σβ", "αχρ", "χρ", "απλ", "αειμν", "δυσχρ", "ευχρ", "κοινοχρ", "παλιμψ"),
false);
@ -610,7 +609,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc18 = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc18 = new CharArraySet(
Arrays.asList("ν", "ρ", "σπι", "στραβομουτσ", "κακομουτσ", "εξων"),
false);
@ -634,7 +633,7 @@ public class GreekStemmer {
return len;
}
private static final CharArraySet exc19 = new CharArraySet(Version.LUCENE_CURRENT,
private static final CharArraySet exc19 = new CharArraySet(
Arrays.asList("παρασουσ", "φ", "χ", "ωριοπλ", "αζ", "αλλοσουσ", "ασουσ"),
false);

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java
View File

@ -30,7 +30,6 @@ import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for English.
@ -57,18 +56,17 @@ public final class EnglishAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #getDefaultStopSet}.
*/
public EnglishAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public EnglishAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public EnglishAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public EnglishAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -76,14 +74,12 @@ public final class EnglishAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public EnglishAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public EnglishAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -101,11 +97,11 @@ public final class EnglishAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new EnglishPossessiveFilter(matchVersion, result);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new EnglishPossessiveFilter(result);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new PorterStemFilter(result);

4
lucene/analysis/common/src/java/org/apache/lucene/analysis/en/EnglishPossessiveFilter.java
View File

@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
/**
* TokenFilter that removes possessives (trailing 's) from words.
@ -30,8 +29,7 @@ import org.apache.lucene.util.Version;
public final class EnglishPossessiveFilter extends TokenFilter {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
// NOTE: version now unused
public EnglishPossessiveFilter(Version version, TokenStream input) {
public EnglishPossessiveFilter(TokenStream input) {
super(input);
}

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/en/EnglishPossessiveFilterFactory.java
View File

@ -39,7 +39,6 @@ public class EnglishPossessiveFilterFactory extends TokenFilterFactory {
/** Creates a new EnglishPossessiveFilterFactory */
public EnglishPossessiveFilterFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@ -47,6 +46,6 @@ public class EnglishPossessiveFilterFactory extends TokenFilterFactory {
@Override
public TokenStream create(TokenStream input) {
return new EnglishPossessiveFilter(luceneMatchVersion, input);
return new EnglishPossessiveFilter(input);
}
}

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/en/KStemmer.java
View File

@ -64,7 +64,6 @@ import org.apache.lucene.analysis.util.OpenStringBuilder;
* <p>Copyright: Copyright 2008, Luicid Imagination, Inc. </p>
* <p>Copyright: Copyright 2003, CIIR University of Massachusetts Amherst (http://ciir.cs.umass.edu) </p>
*/
import org.apache.lucene.util.Version;
/**
* This class implements the Kstem algorithm
@ -280,7 +279,7 @@ public class KStemmer {
DictEntry defaultEntry;
DictEntry entry;
CharArrayMap<DictEntry> d = new CharArrayMap<>(Version.LUCENE_CURRENT, 1000, false);
CharArrayMap<DictEntry> d = new CharArrayMap<>(1000, false);
for (int i = 0; i < exceptionWords.length; i++) {
if (!d.containsKey(exceptionWords[i])) {
entry = new DictEntry(exceptionWords[i], true);

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishAnalyzer.java
View File

@ -34,7 +34,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Spanish.
@ -63,7 +62,7 @@ public final class SpanishAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -75,18 +74,17 @@ public final class SpanishAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public SpanishAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public SpanishAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public SpanishAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public SpanishAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -94,14 +92,12 @@ public final class SpanishAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public SpanishAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public SpanishAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -118,10 +114,10 @@ public final class SpanishAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SpanishLightStemFilter(result);

26
lucene/analysis/common/src/java/org/apache/lucene/analysis/eu/BasqueAnalyzer.java
View File

@ -31,7 +31,6 @@ import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.BasqueStemmer;
/**
@ -73,18 +72,17 @@ public final class BasqueAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public BasqueAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public BasqueAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public BasqueAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public BasqueAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -92,14 +90,12 @@ public final class BasqueAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public BasqueAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public BasqueAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -116,10 +112,10 @@ public final class BasqueAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new BasqueStemmer());

17
lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
View File

@ -29,7 +29,6 @@ import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Persian.
@ -87,20 +86,18 @@ public final class PersianAnalyzer extends StopwordAnalyzerBase {
* Builds an analyzer with the default stop words:
* {@link #DEFAULT_STOPWORD_FILE}.
*/
public PersianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public PersianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
*/
public PersianAnalyzer(Version matchVersion, CharArraySet stopwords){
super(matchVersion, stopwords);
public PersianAnalyzer(CharArraySet stopwords){
super(stopwords);
}
/**
@ -115,8 +112,8 @@ public final class PersianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new LowerCaseFilter(matchVersion, source);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new LowerCaseFilter(source);
result = new ArabicNormalizationFilter(result);
/* additional persian-specific normalization */
result = new PersianNormalizationFilter(result);
@ -124,7 +121,7 @@ public final class PersianAnalyzer extends StopwordAnalyzerBase {
* the order here is important: the stopword list is normalized with the
* above!
*/
return new TokenStreamComponents(source, new StopFilter(matchVersion, result, stopwords));
return new TokenStreamComponents(source, new StopFilter(result, stopwords));
}
/**

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishAnalyzer.java
View File

@ -34,7 +34,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.FinnishStemmer;
/**
@ -64,7 +63,7 @@ public final class FinnishAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -76,18 +75,17 @@ public final class FinnishAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public FinnishAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public FinnishAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public FinnishAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public FinnishAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -95,14 +93,12 @@ public final class FinnishAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public FinnishAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public FinnishAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -119,10 +115,10 @@ public final class FinnishAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new FinnishStemmer());

30
lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
View File

@ -59,7 +59,7 @@ public final class FrenchAnalyzer extends StopwordAnalyzerBase {
/** Default set of articles for ElisionFilter */
public static final CharArraySet DEFAULT_ARTICLES = CharArraySet.unmodifiableSet(
new CharArraySet(Version.LUCENE_CURRENT, Arrays.asList(
new CharArraySet(Arrays.asList(
"l", "m", "t", "qu", "n", "s", "j", "d", "c", "jusqu", "quoiqu", "lorsqu", "puisqu"), true));
/**
@ -80,7 +80,7 @@ public final class FrenchAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -92,37 +92,33 @@ public final class FrenchAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words ({@link #getDefaultStopSet}).
*/
public FrenchAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public FrenchAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
*/
public FrenchAnalyzer(Version matchVersion, CharArraySet stopwords){
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public FrenchAnalyzer(CharArraySet stopwords){
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
* @param stemExclutionSet
* a stemming exclusion set
*/
public FrenchAnalyzer(Version matchVersion, CharArraySet stopwords,
public FrenchAnalyzer(CharArraySet stopwords,
CharArraySet stemExclutionSet) {
super(matchVersion, stopwords);
super(stopwords);
this.excltable = CharArraySet.unmodifiableSet(CharArraySet
.copy(matchVersion, stemExclutionSet));
.copy(stemExclutionSet));
}
/**
@ -139,11 +135,11 @@ public final class FrenchAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!excltable.isEmpty())
result = new SetKeywordMarkerFilter(result, excltable);
result = new FrenchLightStemFilter(result);

30
lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java
View File

@ -32,7 +32,6 @@ import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.ElisionFilter;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.IrishStemmer;
/**
@ -45,7 +44,7 @@ public final class IrishAnalyzer extends StopwordAnalyzerBase {
public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
private static final CharArraySet DEFAULT_ARTICLES = CharArraySet.unmodifiableSet(
new CharArraySet(Version.LUCENE_CURRENT,
new CharArraySet(
Arrays.asList(
"d", "m", "b"
), true));
@ -56,7 +55,7 @@ public final class IrishAnalyzer extends StopwordAnalyzerBase {
* with phrase queries versus tAthair (which would not have a gap).
*/
private static final CharArraySet HYPHENATIONS = CharArraySet.unmodifiableSet(
new CharArraySet(Version.LUCENE_CURRENT,
new CharArraySet(
Arrays.asList(
"h", "n", "t"
), true));
@ -91,18 +90,17 @@ public final class IrishAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public IrishAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public IrishAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public IrishAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public IrishAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -110,14 +108,12 @@ public final class IrishAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public IrishAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public IrishAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -134,12 +130,12 @@ public final class IrishAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new StopFilter(matchVersion, result, HYPHENATIONS);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new StopFilter(result, HYPHENATIONS);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new IrishLowerCaseFilter(result);
result = new StopFilter(matchVersion, result, stopwords);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new IrishStemmer());

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/gl/GalicianAnalyzer.java
View File

@ -33,7 +33,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Galician.
@ -62,7 +61,7 @@ public final class GalicianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getWordSet(IOUtils.getDecodingReader(GalicianAnalyzer.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -74,18 +73,17 @@ public final class GalicianAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public GalicianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public GalicianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public GalicianAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public GalicianAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -93,14 +91,12 @@ public final class GalicianAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public GalicianAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public GalicianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -117,10 +113,10 @@ public final class GalicianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new GalicianStemFilter(result);

24
lucene/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiAnalyzer.java
View File

@ -29,7 +29,6 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.in.IndicNormalizationFilter;
import org.apache.lucene.util.Version;
/**
* Analyzer for Hindi.
@ -75,32 +74,29 @@ public final class HindiAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the given stop words
*
* @param version lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a stemming exclusion set
*/
public HindiAnalyzer(Version version, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(version, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(
CharArraySet.copy(matchVersion, stemExclusionSet));
public HindiAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
* Builds an analyzer with the given stop words
*
* @param version lucene compatibility version
* @param stopwords a stopword set
*/
public HindiAnalyzer(Version version, CharArraySet stopwords) {
this(version, stopwords, CharArraySet.EMPTY_SET);
public HindiAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
* Builds an analyzer with the default stop words:
* {@link #DEFAULT_STOPWORD_FILE}.
*/
public HindiAnalyzer(Version version) {
this(version, DefaultSetHolder.DEFAULT_STOP_SET);
public HindiAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
@ -117,13 +113,13 @@ public final class HindiAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new LowerCaseFilter(matchVersion, source);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new LowerCaseFilter(source);
if (!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new IndicNormalizationFilter(result);
result = new HindiNormalizationFilter(result);
result = new StopFilter(matchVersion, result, stopwords);
result = new StopFilter(result, stopwords);
result = new HindiStemFilter(result);
return new TokenStreamComponents(source, result);
}

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianAnalyzer.java
View File

@ -34,7 +34,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.HungarianStemmer;
/**
@ -64,7 +63,7 @@ public final class HungarianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -76,18 +75,17 @@ public final class HungarianAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public HungarianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public HungarianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public HungarianAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public HungarianAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -95,14 +93,12 @@ public final class HungarianAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public HungarianAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public HungarianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -119,10 +115,10 @@ public final class HungarianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new HungarianStemmer());

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
View File

@ -215,7 +215,7 @@ final class Stemmer {
if (stems.size() < 2) {
return stems;
}
CharArraySet terms = new CharArraySet(Version.LUCENE_CURRENT, 8, dictionary.ignoreCase);
CharArraySet terms = new CharArraySet(8, dictionary.ignoreCase);
List<CharsRef> deduped = new ArrayList<>();
for (CharsRef s : stems) {
if (!terms.contains(s)) {

26
lucene/analysis/common/src/java/org/apache/lucene/analysis/hy/ArmenianAnalyzer.java
View File

@ -31,7 +31,6 @@ import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.ArmenianStemmer;
/**
@ -73,18 +72,17 @@ public final class ArmenianAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public ArmenianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public ArmenianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public ArmenianAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public ArmenianAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -92,14 +90,12 @@ public final class ArmenianAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public ArmenianAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public ArmenianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -116,10 +112,10 @@ public final class ArmenianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new ArmenianStemmer());

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/id/IndonesianAnalyzer.java
View File

@ -29,7 +29,6 @@ import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
/**
* Analyzer for Indonesian (Bahasa)
@ -69,20 +68,18 @@ public final class IndonesianAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public IndonesianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public IndonesianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
*/
public IndonesianAnalyzer(Version matchVersion, CharArraySet stopwords){
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public IndonesianAnalyzer(CharArraySet stopwords){
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -90,17 +87,14 @@ public final class IndonesianAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* {@link IndonesianStemFilter}.
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
* @param stemExclusionSet
* a set of terms not to be stemmed
*/
public IndonesianAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet){
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public IndonesianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet){
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -116,10 +110,10 @@ public final class IndonesianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if (!stemExclusionSet.isEmpty()) {
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
}

30
lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java
View File

@ -36,7 +36,6 @@ import org.apache.lucene.analysis.util.ElisionFilter;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Italian.
@ -48,7 +47,7 @@ public final class ItalianAnalyzer extends StopwordAnalyzerBase {
public final static String DEFAULT_STOPWORD_FILE = "italian_stop.txt";
private static final CharArraySet DEFAULT_ARTICLES = CharArraySet.unmodifiableSet(
new CharArraySet(Version.LUCENE_CURRENT,
new CharArraySet(
Arrays.asList(
"c", "l", "all", "dall", "dell", "nell", "sull", "coll", "pell",
"gl", "agl", "dagl", "degl", "negl", "sugl", "un", "m", "t", "s", "v", "d"
@ -72,7 +71,7 @@ public final class ItalianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -84,18 +83,17 @@ public final class ItalianAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public ItalianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public ItalianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public ItalianAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public ItalianAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -103,14 +101,12 @@ public final class ItalianAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public ItalianAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public ItalianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -127,11 +123,11 @@ public final class ItalianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new ItalianLightStemFilter(result);

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/lv/LatvianAnalyzer.java
View File

@ -33,7 +33,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Latvian.
@ -62,7 +61,7 @@ public final class LatvianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getWordSet(IOUtils.getDecodingReader(LatvianAnalyzer.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -74,18 +73,17 @@ public final class LatvianAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public LatvianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public LatvianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public LatvianAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public LatvianAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -93,14 +91,12 @@ public final class LatvianAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public LatvianAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public LatvianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -117,10 +113,10 @@ public final class LatvianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new LatvianStemFilter(result);

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java
View File

@ -82,7 +82,7 @@ public class CapitalizationFilterFactory extends TokenFilterFactory {
boolean ignoreCase = getBoolean(args, KEEP_IGNORE_CASE, false);
Set<String> k = getSet(args, KEEP);
if (k != null) {
keep = new CharArraySet(luceneMatchVersion, 10, ignoreCase);
keep = new CharArraySet(10, ignoreCase);
keep.addAll(k);
}

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CodepointCountFilter.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.analysis.miscellaneous;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
/**
* Removes words that are too long or too short from the stream.
@ -39,13 +38,12 @@ public final class CodepointCountFilter extends FilteringTokenFilter {
* Create a new {@link CodepointCountFilter}. This will filter out tokens whose
* {@link CharTermAttribute} is either too short ({@link Character#codePointCount(char[], int, int)}
* &lt; min) or too long ({@link Character#codePointCount(char[], int, int)} &gt; max).
* @param version the Lucene match version
* @param in the {@link TokenStream} to consume
* @param min the minimum length
* @param max the maximum length
*/
public CodepointCountFilter(Version version, TokenStream in, int min, int max) {
super(version, in);
public CodepointCountFilter(TokenStream in, int min, int max) {
super(in);
if (min < 0) {
throw new IllegalArgumentException("minimum length must be greater than or equal to zero");
}

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CodepointCountFilterFactory.java
View File

@ -50,6 +50,6 @@ public class CodepointCountFilterFactory extends TokenFilterFactory {
@Override
public CodepointCountFilter create(TokenStream input) {
return new CodepointCountFilter(luceneMatchVersion, input, min, max);
return new CodepointCountFilter(input, min, max);
}
}

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java
View File

@ -21,7 +21,6 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
/**
* A TokenFilter that only keeps tokens with text contained in the
@ -37,12 +36,11 @@ public final class KeepWordFilter extends FilteringTokenFilter {
* Create a new {@link KeepWordFilter}.
* <p><b>NOTE</b>: The words set passed to this constructor will be directly
* used by this filter and should not be modified.
* @param version the Lucene match version
* @param in the {@link TokenStream} to consume
* @param words the words to keep
*/
public KeepWordFilter(Version version, TokenStream in, CharArraySet words) {
super(version, in);
public KeepWordFilter(TokenStream in, CharArraySet words) {
super(in);
this.words = words;
}

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java
View File

@ -44,7 +44,6 @@ public class KeepWordFilterFactory extends TokenFilterFactory implements Resourc
/** Creates a new KeepWordFilterFactory */
public KeepWordFilterFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
wordFiles = get(args, "words");
ignoreCase = getBoolean(args, "ignoreCase", false);
if (!args.isEmpty()) {
@ -73,7 +72,7 @@ public class KeepWordFilterFactory extends TokenFilterFactory implements Resourc
if (words == null) {
return input;
} else {
final TokenStream filter = new KeepWordFilter(luceneMatchVersion, input, words);
final TokenStream filter = new KeepWordFilter(input, words);
return filter;
}
}

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.analysis.miscellaneous;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
/**
* Removes words that are too long or too short from the stream.
@ -39,13 +38,12 @@ public final class LengthFilter extends FilteringTokenFilter {
* Create a new {@link LengthFilter}. This will filter out tokens whose
* {@link CharTermAttribute} is either too short ({@link CharTermAttribute#length()}
* &lt; min) or too long ({@link CharTermAttribute#length()} &gt; max).
* @param version the Lucene match version
* @param in the {@link TokenStream} to consume
* @param min the minimum length
* @param max the maximum length
*/
public LengthFilter(Version version, TokenStream in, int min, int max) {
super(version, in);
public LengthFilter(TokenStream in, int min, int max) {
super(in);
if (min < 0) {
throw new IllegalArgumentException("minimum length must be greater than or equal to zero");
}

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java
View File

@ -50,7 +50,7 @@ public class LengthFilterFactory extends TokenFilterFactory {
@Override
public LengthFilter create(TokenStream input) {
final LengthFilter filter = new LengthFilter(luceneMatchVersion, input,min,max);
final LengthFilter filter = new LengthFilter(input,min,max);
return filter;
}
}

4
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/RemoveDuplicatesTokenFilter.java
View File

@ -22,7 +22,6 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
import java.io.IOException;
@ -34,8 +33,7 @@ public final class RemoveDuplicatesTokenFilter extends TokenFilter {
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncAttribute = addAttribute(PositionIncrementAttribute.class);
// use a fixed version, as we don't care about case sensitivity.
private final CharArraySet previous = new CharArraySet(Version.LUCENE_CURRENT, 8, false);
private final CharArraySet previous = new CharArraySet(8, false);
/**
* Creates a new RemoveDuplicatesTokenFilter

7
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java
View File

@ -20,15 +20,11 @@ package org.apache.lucene.analysis.miscellaneous;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.util.Version;
import java.io.IOException;
/**
* Trims leading and trailing whitespace from Tokens in the stream.
* <p>As of Lucene 4.4, this filter does not support updateOffsets=true anymore
* as it can lead to broken token streams.
*/
public final class TrimFilter extends TokenFilter {
@ -36,10 +32,9 @@ public final class TrimFilter extends TokenFilter {
/**
* Create a new {@link TrimFilter}.
* @param version the Lucene match version
* @param in the stream to consume
*/
public TrimFilter(Version version, TokenStream in) {
public TrimFilter(TokenStream in) {
super(in);
}

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java
View File

@ -47,7 +47,7 @@ public class TrimFilterFactory extends TokenFilterFactory {
@Override
public TrimFilter create(TokenStream input) {
final TrimFilter filter = new TrimFilter(luceneMatchVersion, input);
final TrimFilter filter = new TrimFilter(input);
return filter;
}
}

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
View File

@ -80,7 +80,7 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
}
this.charUtils = version.onOrAfter(Version.LUCENE_4_4)
? CharacterUtils.getInstance(version)
? CharacterUtils.getInstance()
: CharacterUtils.getJava4Instance();
this.minGram = minGram;
this.maxGram = maxGram;

4
lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
View File

@ -81,10 +81,10 @@ public final class NGramTokenFilter extends TokenFilter {
* @param maxGram the largest n-gram to generate
*/
public NGramTokenFilter(Version version, TokenStream input, int minGram, int maxGram) {
super(new CodepointCountFilter(version, input, minGram, Integer.MAX_VALUE));
super(new CodepointCountFilter(input, minGram, Integer.MAX_VALUE));
this.version = version;
this.charUtils = version.onOrAfter(Version.LUCENE_4_4)
? CharacterUtils.getInstance(version)
? CharacterUtils.getInstance()
: CharacterUtils.getJava4Instance();
if (minGram < 1) {
throw new IllegalArgumentException("minGram must be greater than zero");

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
View File

@ -121,7 +121,7 @@ public class NGramTokenizer extends Tokenizer {
throw new IllegalArgumentException("This class only works with Lucene 4.4+. To emulate the old (broken) behavior of NGramTokenizer, use Lucene43NGramTokenizer");
}
charUtils = version.onOrAfter(Version.LUCENE_4_4)
? CharacterUtils.getInstance(version)
? CharacterUtils.getInstance()
: CharacterUtils.getJava4Instance();
if (minGram < 1) {
throw new IllegalArgumentException("minGram must be greater than zero");

38
lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
View File

@ -28,13 +28,11 @@ import org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer; // for javadoc
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.Reader;
@ -50,10 +48,8 @@ import java.nio.charset.StandardCharsets;
* A default set of stopwords is used unless an alternative list is specified, but the
* exclusion list is empty by default.
* </p>
*
* <p><b>NOTE</b>: This class uses the same {@link Version}
* dependent settings as {@link StandardAnalyzer}.</p>
*/
// TODO: extend StopwordAnalyzerBase
public final class DutchAnalyzer extends Analyzer {
/** File containing default Dutch stopwords. */
@ -73,14 +69,14 @@ public final class DutchAnalyzer extends Analyzer {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
throw new RuntimeException("Unable to load default stopword set");
}
DEFAULT_STEM_DICT = new CharArrayMap<>(Version.LUCENE_CURRENT, 4, false);
DEFAULT_STEM_DICT = new CharArrayMap<>(4, false);
DEFAULT_STEM_DICT.put("fiets", "fiets"); //otherwise fiet
DEFAULT_STEM_DICT.put("bromfiets", "bromfiets"); //otherwise bromfiet
DEFAULT_STEM_DICT.put("ei", "eier");
@ -100,29 +96,27 @@ public final class DutchAnalyzer extends Analyzer {
private CharArraySet excltable = CharArraySet.EMPTY_SET;
private final StemmerOverrideMap stemdict;
private final Version matchVersion;
/**
* Builds an analyzer with the default stop words ({@link #getDefaultStopSet()})
* and a few default entries for the stem exclusion table.
*
*/
public DutchAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET, CharArraySet.EMPTY_SET, DefaultSetHolder.DEFAULT_STEM_DICT);
public DutchAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET, CharArraySet.EMPTY_SET, DefaultSetHolder.DEFAULT_STEM_DICT);
}
public DutchAnalyzer(Version matchVersion, CharArraySet stopwords){
this(matchVersion, stopwords, CharArraySet.EMPTY_SET, DefaultSetHolder.DEFAULT_STEM_DICT);
public DutchAnalyzer(CharArraySet stopwords){
this(stopwords, CharArraySet.EMPTY_SET, DefaultSetHolder.DEFAULT_STEM_DICT);
}
public DutchAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionTable){
this(matchVersion, stopwords, stemExclusionTable, DefaultSetHolder.DEFAULT_STEM_DICT);
public DutchAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionTable){
this(stopwords, stemExclusionTable, DefaultSetHolder.DEFAULT_STEM_DICT);
}
public DutchAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionTable, CharArrayMap<String> stemOverrideDict) {
this.matchVersion = matchVersion;
this.stoptable = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion, stopwords));
this.excltable = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion, stemExclusionTable));
public DutchAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionTable, CharArrayMap<String> stemOverrideDict) {
this.stoptable = CharArraySet.unmodifiableSet(CharArraySet.copy(stopwords));
this.excltable = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionTable));
if (stemOverrideDict.isEmpty()) {
this.stemdict = null;
} else {
@ -154,10 +148,10 @@ public final class DutchAnalyzer extends Analyzer {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stoptable);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stoptable);
if (!excltable.isEmpty())
result = new SetKeywordMarkerFilter(result, excltable);
if (stemdict != null)

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianAnalyzer.java
View File

@ -34,7 +34,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.NorwegianStemmer;
/**
@ -64,7 +63,7 @@ public final class NorwegianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -76,18 +75,17 @@ public final class NorwegianAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public NorwegianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public NorwegianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public NorwegianAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public NorwegianAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -95,14 +93,12 @@ public final class NorwegianAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public NorwegianAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public NorwegianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -119,10 +115,10 @@ public final class NorwegianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new NorwegianStemmer());

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseAnalyzer.java
View File

@ -34,7 +34,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Portuguese.
@ -63,7 +62,7 @@ public final class PortugueseAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -75,18 +74,17 @@ public final class PortugueseAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public PortugueseAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public PortugueseAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public PortugueseAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public PortugueseAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -94,14 +92,12 @@ public final class PortugueseAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public PortugueseAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public PortugueseAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -118,10 +114,10 @@ public final class PortugueseAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new PortugueseLightStemFilter(result);

4
lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java
View File

@ -31,7 +31,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
import static org.apache.lucene.analysis.util.StemmerUtil.*;
@ -135,8 +134,7 @@ public abstract class RSLPStemmerBase {
if (!exceptions[i].endsWith(suffix))
throw new RuntimeException("useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
}
this.exceptions = new CharArraySet(Version.LUCENE_CURRENT,
Arrays.asList(exceptions), false);
this.exceptions = new CharArraySet(Arrays.asList(exceptions), false);
}
@Override

25
lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
View File

@ -31,7 +31,6 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.Version;
/**
* An {@link Analyzer} used primarily at query time to wrap another analyzer and provide a layer of protection
@ -50,23 +49,20 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
//The default maximum percentage (40%) of index documents which
//can contain a term, after which the term is considered to be a stop word.
public static final float defaultMaxDocFreqPercent = 0.4f;
private final Version matchVersion;
/**
* Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for all
* indexed fields from terms with a document frequency percentage greater than
* {@link #defaultMaxDocFreqPercent}
*
* @param matchVersion Version to be used in {@link StopFilter}
* @param delegate Analyzer whose TokenStream will be filtered
* @param indexReader IndexReader to identify the stopwords from
* @throws IOException Can be thrown while reading from the IndexReader
*/
public QueryAutoStopWordAnalyzer(
Version matchVersion,
Analyzer delegate,
IndexReader indexReader) throws IOException {
this(matchVersion, delegate, indexReader, defaultMaxDocFreqPercent);
this(delegate, indexReader, defaultMaxDocFreqPercent);
}
/**
@ -74,18 +70,16 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
* indexed fields from terms with a document frequency greater than the given
* maxDocFreq
*
* @param matchVersion Version to be used in {@link StopFilter}
* @param delegate Analyzer whose TokenStream will be filtered
* @param indexReader IndexReader to identify the stopwords from
* @param maxDocFreq Document frequency terms should be above in order to be stopwords
* @throws IOException Can be thrown while reading from the IndexReader
*/
public QueryAutoStopWordAnalyzer(
Version matchVersion,
Analyzer delegate,
IndexReader indexReader,
int maxDocFreq) throws IOException {
this(matchVersion, delegate, indexReader, MultiFields.getIndexedFields(indexReader), maxDocFreq);
this(delegate, indexReader, MultiFields.getIndexedFields(indexReader), maxDocFreq);
}
/**
@ -93,7 +87,6 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
* indexed fields from terms with a document frequency percentage greater than
* the given maxPercentDocs
*
* @param matchVersion Version to be used in {@link StopFilter}
* @param delegate Analyzer whose TokenStream will be filtered
* @param indexReader IndexReader to identify the stopwords from
* @param maxPercentDocs The maximum percentage (between 0.0 and 1.0) of index documents which
@ -101,11 +94,10 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
* @throws IOException Can be thrown while reading from the IndexReader
*/
public QueryAutoStopWordAnalyzer(
Version matchVersion,
Analyzer delegate,
IndexReader indexReader,
float maxPercentDocs) throws IOException {
this(matchVersion, delegate, indexReader, MultiFields.getIndexedFields(indexReader), maxPercentDocs);
this(delegate, indexReader, MultiFields.getIndexedFields(indexReader), maxPercentDocs);
}
/**
@ -113,7 +105,6 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
* given selection of fields from terms with a document frequency percentage
* greater than the given maxPercentDocs
*
* @param matchVersion Version to be used in {@link StopFilter}
* @param delegate Analyzer whose TokenStream will be filtered
* @param indexReader IndexReader to identify the stopwords from
* @param fields Selection of fields to calculate stopwords for
@ -122,12 +113,11 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
* @throws IOException Can be thrown while reading from the IndexReader
*/
public QueryAutoStopWordAnalyzer(
Version matchVersion,
Analyzer delegate,
IndexReader indexReader,
Collection<String> fields,
float maxPercentDocs) throws IOException {
this(matchVersion, delegate, indexReader, fields, (int) (indexReader.numDocs() * maxPercentDocs));
this(delegate, indexReader, fields, (int) (indexReader.numDocs() * maxPercentDocs));
}
/**
@ -135,7 +125,6 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
* given selection of fields from terms with a document frequency greater than
* the given maxDocFreq
*
* @param matchVersion Version to be used in {@link StopFilter}
* @param delegate Analyzer whose TokenStream will be filtered
* @param indexReader IndexReader to identify the stopwords from
* @param fields Selection of fields to calculate stopwords for
@ -143,13 +132,11 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
* @throws IOException Can be thrown while reading from the IndexReader
*/
public QueryAutoStopWordAnalyzer(
Version matchVersion,
Analyzer delegate,
IndexReader indexReader,
Collection<String> fields,
int maxDocFreq) throws IOException {
super(delegate.getReuseStrategy());
this.matchVersion = matchVersion;
this.delegate = delegate;
for (String field : fields) {
@ -181,8 +168,8 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
if (stopWords == null) {
return components;
}
StopFilter stopFilter = new StopFilter(matchVersion, components.getTokenStream(),
new CharArraySet(matchVersion, stopWords, false));
StopFilter stopFilter = new StopFilter(components.getTokenStream(),
new CharArraySet(stopWords, false));
return new TokenStreamComponents(components.getTokenizer(), stopFilter);
}

32
lucene/analysis/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.analysis.reverse;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
import java.io.IOException;
@ -36,7 +35,6 @@ public final class ReverseStringFilter extends TokenFilter {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final char marker;
private final Version matchVersion;
private static final char NOMARKER = '\uFFFF';
/**
@ -66,11 +64,10 @@ public final class ReverseStringFilter extends TokenFilter {
* The reversed tokens will not be marked.
* </p>
*
* @param matchVersion Lucene compatibility version
* @param in {@link TokenStream} to filter
*/
public ReverseStringFilter(Version matchVersion, TokenStream in) {
this(matchVersion, in, NOMARKER);
public ReverseStringFilter(TokenStream in) {
this(in, NOMARKER);
}
/**
@ -81,13 +78,11 @@ public final class ReverseStringFilter extends TokenFilter {
* character.
* </p>
*
* @param matchVersion compatibility version
* @param in {@link TokenStream} to filter
* @param marker A character used to mark reversed tokens
*/
public ReverseStringFilter(Version matchVersion, TokenStream in, char marker) {
public ReverseStringFilter(TokenStream in, char marker) {
super(in);
this.matchVersion = matchVersion;
this.marker = marker;
}
@ -100,7 +95,7 @@ public final class ReverseStringFilter extends TokenFilter {
termAtt.resizeBuffer(len);
termAtt.buffer()[len - 1] = marker;
}
reverse( matchVersion, termAtt.buffer(), 0, len );
reverse( termAtt.buffer(), 0, len );
termAtt.setLength(len);
return true;
} else {
@ -111,48 +106,43 @@ public final class ReverseStringFilter extends TokenFilter {
/**
* Reverses the given input string
*
* @param matchVersion compatibility version
* @param input the string to reverse
* @return the given input string in reversed order
*/
public static String reverse( Version matchVersion, final String input ){
public static String reverse(final String input ){
final char[] charInput = input.toCharArray();
reverse( matchVersion, charInput, 0, charInput.length );
reverse( charInput, 0, charInput.length );
return new String( charInput );
}
/**
* Reverses the given input buffer in-place
* @param matchVersion compatibility version
* @param buffer the input char array to reverse
*/
public static void reverse(Version matchVersion, final char[] buffer) {
reverse(matchVersion, buffer, 0, buffer.length);
public static void reverse(final char[] buffer) {
reverse(buffer, 0, buffer.length);
}
/**
* Partially reverses the given input buffer in-place from offset 0
* up to the given length.
* @param matchVersion compatibility version
* @param buffer the input char array to reverse
* @param len the length in the buffer up to where the
* buffer should be reversed
*/
public static void reverse(Version matchVersion, final char[] buffer,
final int len) {
reverse( matchVersion, buffer, 0, len );
public static void reverse(final char[] buffer, final int len) {
reverse( buffer, 0, len );
}
/**
* Partially reverses the given input buffer in-place from the given offset
* up to the given length.
* @param matchVersion compatibility version
* @param buffer the input char array to reverse
* @param start the offset from where to reverse the buffer
* @param len the length in the buffer up to where the
* buffer should be reversed
*/
public static void reverse(Version matchVersion, final char[] buffer,
public static void reverse(final char[] buffer,
final int start, final int len) {
/* modified version of Apache Harmony AbstractStringBuilder reverse0() */
if (len < 2)

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilterFactory.java
View File

@ -40,7 +40,6 @@ public class ReverseStringFilterFactory extends TokenFilterFactory {
/** Creates a new ReverseStringFilterFactory */
public ReverseStringFilterFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@ -48,7 +47,7 @@ public class ReverseStringFilterFactory extends TokenFilterFactory {
@Override
public ReverseStringFilter create(TokenStream in) {
return new ReverseStringFilter(luceneMatchVersion,in);
return new ReverseStringFilter(in);
}
}

25
lucene/analysis/common/src/java/org/apache/lucene/analysis/ro/RomanianAnalyzer.java
View File

@ -78,18 +78,17 @@ public final class RomanianAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public RomanianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public RomanianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public RomanianAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public RomanianAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -97,14 +96,12 @@ public final class RomanianAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public RomanianAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public RomanianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -121,10 +118,10 @@ public final class RomanianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new RomanianStemmer());

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
View File

@ -54,7 +54,7 @@ public final class RussianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -74,34 +74,30 @@ public final class RussianAnalyzer extends StopwordAnalyzerBase {
return DefaultSetHolder.DEFAULT_STOP_SET;
}
public RussianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public RussianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
*/
public RussianAnalyzer(Version matchVersion, CharArraySet stopwords){
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public RussianAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
* Builds an analyzer with the given stop words
*
* @param matchVersion
* lucene compatibility version
* @param stopwords
* a stopword set
* @param stemExclusionSet a set of words not to be stemmed
*/
public RussianAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet){
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion, stemExclusionSet));
public RussianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -117,10 +113,10 @@ public final class RussianAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if (!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new org.tartarus.snowball.ext.RussianStemmer());

9
lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.analysis.shingle;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.AnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.util.Version;
/**
* A ShingleAnalyzerWrapper wraps a {@link ShingleFilter} around another {@link Analyzer}.
@ -101,15 +100,15 @@ public final class ShingleAnalyzerWrapper extends AnalyzerWrapper {
/**
* Wraps {@link StandardAnalyzer}.
*/
public ShingleAnalyzerWrapper(Version matchVersion) {
this(matchVersion, ShingleFilter.DEFAULT_MIN_SHINGLE_SIZE, ShingleFilter.DEFAULT_MAX_SHINGLE_SIZE);
public ShingleAnalyzerWrapper() {
this(ShingleFilter.DEFAULT_MIN_SHINGLE_SIZE, ShingleFilter.DEFAULT_MAX_SHINGLE_SIZE);
}
/**
* Wraps {@link StandardAnalyzer}.
*/
public ShingleAnalyzerWrapper(Version matchVersion, int minShingleSize, int maxShingleSize) {
this(new StandardAnalyzer(matchVersion), minShingleSize, maxShingleSize);
public ShingleAnalyzerWrapper(int minShingleSize, int maxShingleSize) {
this(new StandardAnalyzer(), minShingleSize, maxShingleSize);
}
/**

42
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
View File

@ -17,16 +17,14 @@ package org.apache.lucene.analysis.standard;
* limitations under the License.
*/
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.Version;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
@ -34,18 +32,6 @@ import java.io.Reader;
* Filters {@link ClassicTokenizer} with {@link ClassicFilter}, {@link
* LowerCaseFilter} and {@link StopFilter}, using a list of
* English stop words.
*
* <a name="version"/>
* <p>You must specify the required {@link Version}
* compatibility when creating ClassicAnalyzer:
* <ul>
* <li> As of 3.1, StopFilter correctly handles Unicode 4.0
* supplementary characters in stopwords
* <li> As of 2.9, StopFilter preserves position
* increments
* <li> As of 2.4, Tokens incorrectly identified as acronyms
* are corrected (see <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1068</a>)
* </ul>
*
* ClassicAnalyzer was named StandardAnalyzer in Lucene versions prior to 3.1.
* As of 3.1, {@link StandardAnalyzer} implements Unicode text segmentation,
@ -63,29 +49,23 @@ public final class ClassicAnalyzer extends StopwordAnalyzerBase {
public static final CharArraySet STOP_WORDS_SET = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
/** Builds an analyzer with the given stop words.
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @param stopWords stop words */
public ClassicAnalyzer(Version matchVersion, CharArraySet stopWords) {
super(matchVersion, stopWords);
public ClassicAnalyzer(CharArraySet stopWords) {
super(stopWords);
}
/** Builds an analyzer with the default stop words ({@link
* #STOP_WORDS_SET}).
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
*/
public ClassicAnalyzer(Version matchVersion) {
this(matchVersion, STOP_WORDS_SET);
public ClassicAnalyzer() {
this(STOP_WORDS_SET);
}
/** Builds an analyzer with the stop words from the given reader.
* @see WordlistLoader#getWordSet(Reader, Version)
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @see WordlistLoader#getWordSet(Reader)
* @param stopwords Reader to read stop words from */
public ClassicAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
this(matchVersion, loadStopwordSet(stopwords, matchVersion));
public ClassicAnalyzer(Reader stopwords) throws IOException {
this(loadStopwordSet(stopwords));
}
/**
@ -107,11 +87,11 @@ public final class ClassicAnalyzer extends StopwordAnalyzerBase {
@Override
protected TokenStreamComponents createComponents(final String fieldName) {
final ClassicTokenizer src = new ClassicTokenizer(matchVersion);
final ClassicTokenizer src = new ClassicTokenizer();
src.setMaxTokenLength(maxTokenLength);
TokenStream tok = new ClassicFilter(src);
tok = new LowerCaseFilter(matchVersion, tok);
tok = new StopFilter(matchVersion, tok, stopwords);
tok = new LowerCaseFilter(tok);
tok = new StopFilter(tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
protected void setReader(final Reader reader) throws IOException {

13
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
View File

@ -18,7 +18,6 @@
package org.apache.lucene.analysis.standard;
import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
@ -26,8 +25,6 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/** A grammar-based tokenizer constructed with JFlex
*
@ -102,19 +99,19 @@ public final class ClassicTokenizer extends Tokenizer {
*
* See http://issues.apache.org/jira/browse/LUCENE-1068
*/
public ClassicTokenizer(Version matchVersion) {
init(matchVersion);
public ClassicTokenizer() {
init();
}
/**
* Creates a new ClassicTokenizer with a given {@link org.apache.lucene.util.AttributeFactory}
*/
public ClassicTokenizer(Version matchVersion, AttributeFactory factory) {
public ClassicTokenizer(AttributeFactory factory) {
super(factory);
init(matchVersion);
init();
}
private void init(Version matchVersion) {
private void init() {
this.scanner = new ClassicTokenizerImpl(input);
}

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java
View File

@ -37,7 +37,6 @@ public class ClassicTokenizerFactory extends TokenizerFactory {
/** Creates a new ClassicTokenizerFactory */
public ClassicTokenizerFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
maxTokenLength = getInt(args, "maxTokenLength", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
@ -46,7 +45,7 @@ public class ClassicTokenizerFactory extends TokenizerFactory {
@Override
public ClassicTokenizer create(AttributeFactory factory) {
ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, factory);
ClassicTokenizer tokenizer = new ClassicTokenizer(factory);
tokenizer.setMaxTokenLength(maxTokenLength);
return tokenizer;
}

54
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
View File

@ -17,16 +17,14 @@ package org.apache.lucene.analysis.standard;
* limitations under the License.
*/
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.Version;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
@ -34,26 +32,9 @@ import java.io.Reader;
* Filters {@link StandardTokenizer} with {@link StandardFilter}, {@link
* LowerCaseFilter} and {@link StopFilter}, using a list of
* English stop words.
*
* <a name="version"/>
* <p>You must specify the required {@link Version}
* compatibility when creating StandardAnalyzer:
* <ul>
* <li> As of 3.4, Hiragana and Han characters are no longer wrongly split
* from their combining characters. If you use a previous version number,
* you get the exact broken behavior for backwards compatibility.
* <li> As of 3.1, StandardTokenizer implements Unicode text segmentation,
* and StopFilter correctly handles Unicode 4.0 supplementary characters
* in stopwords. {@link ClassicTokenizer} and {@link ClassicAnalyzer}
* are the pre-3.1 implementations of StandardTokenizer and
* StandardAnalyzer.
* <li> As of 2.9, StopFilter preserves position increments
* <li> As of 2.4, Tokens incorrectly identified as acronyms
* are corrected (see <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1068</a>)
* </ul>
*/
public final class StandardAnalyzer extends StopwordAnalyzerBase {
/** Default maximum allowed token length */
public static final int DEFAULT_MAX_TOKEN_LENGTH = 255;
@ -64,29 +45,22 @@ public final class StandardAnalyzer extends StopwordAnalyzerBase {
public static final CharArraySet STOP_WORDS_SET = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
/** Builds an analyzer with the given stop words.
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @param stopWords stop words */
public StandardAnalyzer(Version matchVersion, CharArraySet stopWords) {
super(matchVersion, stopWords);
public StandardAnalyzer(CharArraySet stopWords) {
super(stopWords);
}
/** Builds an analyzer with the default stop words ({@link
* #STOP_WORDS_SET}).
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
/** Builds an analyzer with the default stop words ({@link #STOP_WORDS_SET}).
*/
public StandardAnalyzer(Version matchVersion) {
this(matchVersion, STOP_WORDS_SET);
public StandardAnalyzer() {
this(STOP_WORDS_SET);
}
/** Builds an analyzer with the stop words from the given reader.
* @see WordlistLoader#getWordSet(Reader, Version)
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @see WordlistLoader#getWordSet(Reader)
* @param stopwords Reader to read stop words from */
public StandardAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
this(matchVersion, loadStopwordSet(stopwords, matchVersion));
public StandardAnalyzer(Reader stopwords) throws IOException {
this(loadStopwordSet(stopwords));
}
/**
@ -108,11 +82,11 @@ public final class StandardAnalyzer extends StopwordAnalyzerBase {
@Override
protected TokenStreamComponents createComponents(final String fieldName) {
final StandardTokenizer src = new StandardTokenizer(matchVersion);
final StandardTokenizer src = new StandardTokenizer();
src.setMaxTokenLength(maxTokenLength);
TokenStream tok = new StandardFilter(matchVersion, src);
tok = new LowerCaseFilter(matchVersion, tok);
tok = new StopFilter(matchVersion, tok, stopwords);
TokenStream tok = new StandardFilter(src);
tok = new LowerCaseFilter(tok);
tok = new StopFilter(tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
protected void setReader(final Reader reader) throws IOException {

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardFilter.java
View File

@ -21,14 +21,13 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.util.Version;
/**
* Normalizes tokens extracted with {@link StandardTokenizer}.
*/
public class StandardFilter extends TokenFilter {
public StandardFilter(Version matchVersion, TokenStream in) {
public StandardFilter(TokenStream in) {
super(in);
}

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardFilterFactory.java
View File

@ -38,7 +38,6 @@ public class StandardFilterFactory extends TokenFilterFactory {
/** Creates a new StandardFilterFactory */
public StandardFilterFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@ -46,6 +45,6 @@ public class StandardFilterFactory extends TokenFilterFactory {
@Override
public StandardFilter create(TokenStream input) {
return new StandardFilter(luceneMatchVersion, input);
return new StandardFilter(input);
}
}

13
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
View File

@ -18,7 +18,6 @@
package org.apache.lucene.analysis.standard;
import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@ -26,8 +25,6 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/** A grammar-based tokenizer constructed with JFlex.
* <p>
@ -116,19 +113,19 @@ public final class StandardTokenizer extends Tokenizer {
* See http://issues.apache.org/jira/browse/LUCENE-1068
*/
public StandardTokenizer(Version matchVersion) {
init(matchVersion);
public StandardTokenizer() {
init();
}
/**
* Creates a new StandardTokenizer with a given {@link org.apache.lucene.util.AttributeFactory}
*/
public StandardTokenizer(Version matchVersion, AttributeFactory factory) {
public StandardTokenizer(AttributeFactory factory) {
super(factory);
init(matchVersion);
init();
}
private void init(Version matchVersion) {
private void init() {
this.scanner = new StandardTokenizerImpl(input);
}

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java
View File

@ -37,7 +37,6 @@ public class StandardTokenizerFactory extends TokenizerFactory {
/** Creates a new StandardTokenizerFactory */
public StandardTokenizerFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
maxTokenLength = getInt(args, "maxTokenLength", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
@ -46,7 +45,7 @@ public class StandardTokenizerFactory extends TokenizerFactory {
@Override
public StandardTokenizer create(AttributeFactory factory) {
StandardTokenizer tokenizer = new StandardTokenizer(luceneMatchVersion, factory);
StandardTokenizer tokenizer = new StandardTokenizer(factory);
tokenizer.setMaxTokenLength(maxTokenLength);
return tokenizer;
}

36
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
View File

@ -34,15 +34,9 @@ import java.io.Reader;
* {@link org.apache.lucene.analysis.core.LowerCaseFilter} and
* {@link org.apache.lucene.analysis.core.StopFilter}, using a list of
* English stop words.
*
* <a name="version"/>
* <p>
* You must specify the required {@link org.apache.lucene.util.Version}
* compatibility when creating UAX29URLEmailAnalyzer
* </p>
*/
public final class UAX29URLEmailAnalyzer extends StopwordAnalyzerBase {
/** Default maximum allowed token length */
public static final int DEFAULT_MAX_TOKEN_LENGTH = StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;
@ -53,29 +47,23 @@ public final class UAX29URLEmailAnalyzer extends StopwordAnalyzerBase {
public static final CharArraySet STOP_WORDS_SET = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
/** Builds an analyzer with the given stop words.
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @param stopWords stop words */
public UAX29URLEmailAnalyzer(Version matchVersion, CharArraySet stopWords) {
super(matchVersion, stopWords);
public UAX29URLEmailAnalyzer(CharArraySet stopWords) {
super(stopWords);
}
/** Builds an analyzer with the default stop words ({@link
* #STOP_WORDS_SET}).
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
*/
public UAX29URLEmailAnalyzer(Version matchVersion) {
this(matchVersion, STOP_WORDS_SET);
public UAX29URLEmailAnalyzer() {
this(STOP_WORDS_SET);
}
/** Builds an analyzer with the stop words from the given reader.
* @see org.apache.lucene.analysis.util.WordlistLoader#getWordSet(java.io.Reader, org.apache.lucene.util.Version)
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @see org.apache.lucene.analysis.util.WordlistLoader#getWordSet(java.io.Reader)
* @param stopwords Reader to read stop words from */
public UAX29URLEmailAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
this(matchVersion, loadStopwordSet(stopwords, matchVersion));
public UAX29URLEmailAnalyzer(Reader stopwords) throws IOException {
this(loadStopwordSet(stopwords));
}
/**
@ -97,11 +85,11 @@ public final class UAX29URLEmailAnalyzer extends StopwordAnalyzerBase {
@Override
protected TokenStreamComponents createComponents(final String fieldName) {
final UAX29URLEmailTokenizer src = new UAX29URLEmailTokenizer(matchVersion);
final UAX29URLEmailTokenizer src = new UAX29URLEmailTokenizer();
src.setMaxTokenLength(maxTokenLength);
TokenStream tok = new StandardFilter(matchVersion, src);
tok = new LowerCaseFilter(matchVersion, tok);
tok = new StopFilter(matchVersion, tok, stopwords);
TokenStream tok = new StandardFilter(src);
tok = new LowerCaseFilter(tok);
tok = new StopFilter(tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
protected void setReader(final Reader reader) throws IOException {

15
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
View File

@ -18,9 +18,6 @@ package org.apache.lucene.analysis.standard;
*/
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
@ -28,8 +25,6 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
* This class implements Word Break rules from the Unicode Text Segmentation
@ -100,19 +95,19 @@ public final class UAX29URLEmailTokenizer extends Tokenizer {
* the <code>input</code> to the newly created JFlex scanner.
*/
public UAX29URLEmailTokenizer(Version matchVersion) {
this.scanner = getScannerFor(matchVersion);
public UAX29URLEmailTokenizer() {
this.scanner = getScanner();
}
/**
* Creates a new UAX29URLEmailTokenizer with a given {@link AttributeFactory}
*/
public UAX29URLEmailTokenizer(Version matchVersion, AttributeFactory factory) {
public UAX29URLEmailTokenizer(AttributeFactory factory) {
super(factory);
this.scanner = getScannerFor(matchVersion);
this.scanner = getScanner();
}
private StandardTokenizerInterface getScannerFor(Version matchVersion) {
private StandardTokenizerInterface getScanner() {
return new UAX29URLEmailTokenizerImpl(input);
}

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java
View File

@ -38,7 +38,6 @@ public class UAX29URLEmailTokenizerFactory extends TokenizerFactory {
/** Creates a new UAX29URLEmailTokenizerFactory */
public UAX29URLEmailTokenizerFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
maxTokenLength = getInt(args, "maxTokenLength", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
@ -47,7 +46,7 @@ public class UAX29URLEmailTokenizerFactory extends TokenizerFactory {
@Override
public UAX29URLEmailTokenizer create(AttributeFactory factory) {
UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, factory);
UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(factory);
tokenizer.setMaxTokenLength(maxTokenLength);
return tokenizer;
}

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishAnalyzer.java
View File

@ -34,7 +34,6 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.SwedishStemmer;
/**
@ -64,7 +63,7 @@ public final class SwedishAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(SnowballFilter.class,
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), Version.LUCENE_CURRENT);
DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@ -76,18 +75,17 @@ public final class SwedishAnalyzer extends StopwordAnalyzerBase {
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
public SwedishAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
public SwedishAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
*/
public SwedishAnalyzer(Version matchVersion, CharArraySet stopwords) {
this(matchVersion, stopwords, CharArraySet.EMPTY_SET);
public SwedishAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
@ -95,14 +93,12 @@ public final class SwedishAnalyzer extends StopwordAnalyzerBase {
* provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
public SwedishAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
public SwedishAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -119,10 +115,10 @@ public final class SwedishAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new SwedishStemmer());

11
lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java
View File

@ -134,8 +134,8 @@ public class SynonymFilterFactory extends TokenFilterFactory implements Resource
analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(Version.LUCENE_CURRENT) : factory.create();
TokenStream stream = ignoreCase ? new LowerCaseFilter(Version.LUCENE_CURRENT, tokenizer) : tokenizer;
Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer() : factory.create();
TokenStream stream = ignoreCase ? new LowerCaseFilter(tokenizer) : tokenizer;
return new TokenStreamComponents(tokenizer, stream);
}
};
@ -202,7 +202,12 @@ public class SynonymFilterFactory extends TokenFilterFactory implements Resource
private Analyzer loadAnalyzer(ResourceLoader loader, String cname) throws IOException {
Class<? extends Analyzer> clazz = loader.findClass(cname, Analyzer.class);
try {
Analyzer analyzer = clazz.getConstructor(Version.class).newInstance(Version.LUCENE_CURRENT);
Analyzer analyzer = null;
try {
analyzer = clazz.getConstructor().newInstance();
} catch (NoSuchMethodException e) {
analyzer = clazz.getConstructor(Version.class).newInstance(Version.LUCENE_CURRENT);
}
if (analyzer instanceof ResourceLoaderAware) {
((ResourceLoaderAware) analyzer).inform(loader);
}

19
lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
View File

@ -23,7 +23,6 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
@ -35,6 +34,7 @@ import org.apache.lucene.util.Version;
* {@link Analyzer} for Thai language. It uses {@link java.text.BreakIterator} to break words.
*/
public final class ThaiAnalyzer extends StopwordAnalyzerBase {
private final Version matchVersion;
/** File containing default Thai stopwords. */
public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
@ -87,7 +87,8 @@ public final class ThaiAnalyzer extends StopwordAnalyzerBase {
* @param stopwords a stopword set
*/
public ThaiAnalyzer(Version matchVersion, CharArraySet stopwords) {
super(matchVersion, stopwords);
super(stopwords);
this.matchVersion = matchVersion;
}
/**
@ -104,15 +105,15 @@ public final class ThaiAnalyzer extends StopwordAnalyzerBase {
protected TokenStreamComponents createComponents(String fieldName) {
if (matchVersion.onOrAfter(Version.LUCENE_4_8)) {
final Tokenizer source = new ThaiTokenizer();
TokenStream result = new LowerCaseFilter(matchVersion, source);
result = new StopFilter(matchVersion, result, stopwords);
TokenStream result = new LowerCaseFilter(source);
result = new StopFilter(result, stopwords);
return new TokenStreamComponents(source, result);
} else {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new ThaiWordFilter(matchVersion, result);
return new TokenStreamComponents(source, new StopFilter(matchVersion, result, stopwords));
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(result);
result = new ThaiWordFilter(result);
return new TokenStreamComponents(source, new StopFilter(result, stopwords));
}
}
}

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java
View File

@ -28,7 +28,6 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.util.CharArrayIterator;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
* {@link TokenFilter} that use {@link java.text.BreakIterator} to break each
@ -61,7 +60,7 @@ public final class ThaiWordFilter extends TokenFilter {
private boolean hasIllegalOffsets = false; // only if the length changed before this filter
/** Creates a new ThaiWordFilter with the specified match version. */
public ThaiWordFilter(Version matchVersion, TokenStream input) {
public ThaiWordFilter(TokenStream input) {
super(input);
if (!DBBI_AVAILABLE)
throw new UnsupportedOperationException("This JRE does not have support for Thai segmentation");

3
lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilterFactory.java
View File

@ -41,7 +41,6 @@ public class ThaiWordFilterFactory extends TokenFilterFactory {
/** Creates a new ThaiWordFilterFactory */
public ThaiWordFilterFactory(Map<String,String> args) {
super(args);
assureMatchVersion();
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@ -49,7 +48,7 @@ public class ThaiWordFilterFactory extends TokenFilterFactory {
@Override
public ThaiWordFilter create(TokenStream input) {
return new ThaiWordFilter(luceneMatchVersion, input);
return new ThaiWordFilter(input);
}
}

13
lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java
View File

@ -38,6 +38,7 @@ import org.tartarus.snowball.ext.TurkishStemmer;
*/
public final class TurkishAnalyzer extends StopwordAnalyzerBase {
private final CharArraySet stemExclusionSet;
private final Version matchVersion;
/** File containing default Turkish stopwords. */
public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
@ -101,9 +102,9 @@ public final class TurkishAnalyzer extends StopwordAnalyzerBase {
* @param stemExclusionSet a set of terms not to be stemmed
*/
public TurkishAnalyzer(Version matchVersion, CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(matchVersion, stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(
matchVersion, stemExclusionSet));
super(stopwords);
this.matchVersion = matchVersion;
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
@ -120,12 +121,12 @@ public final class TurkishAnalyzer extends StopwordAnalyzerBase {
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
final Tokenizer source = new StandardTokenizer();
TokenStream result = new StandardFilter(source);
if(matchVersion.onOrAfter(Version.LUCENE_4_8))
result = new ApostropheFilter(result);
result = new TurkishLowerCaseFilter(result);
result = new StopFilter(matchVersion, result, stopwords);
result = new StopFilter(result, stopwords);
if(!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SnowballFilter(result, new TurkishStemmer());

9
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java
View File

@ -238,12 +238,10 @@ public abstract class AbstractAnalysisFactory {
if (files.size() > 0) {
// default stopwords list has 35 or so words, but maybe don't make it that
// big to start
words = new CharArraySet(luceneMatchVersion,
files.size() * 10, ignoreCase);
words = new CharArraySet(files.size() * 10, ignoreCase);
for (String file : files) {
List<String> wlist = getLines(loader, file.trim());
words.addAll(StopFilter.makeStopSet(luceneMatchVersion, wlist,
ignoreCase));
words.addAll(StopFilter.makeStopSet(wlist, ignoreCase));
}
}
return words;
@ -266,8 +264,7 @@ public abstract class AbstractAnalysisFactory {
if (files.size() > 0) {
// default stopwords list has 35 or so words, but maybe don't make it that
// big to start
words = new CharArraySet(luceneMatchVersion,
files.size() * 10, ignoreCase);
words = new CharArraySet(files.size() * 10, ignoreCase);
for (String file : files) {
InputStream stream = null;
Reader reader = null;

53
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArrayMap.java
View File

@ -25,8 +25,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.util.CharacterUtils;
import org.apache.lucene.util.Version;
/**
* A simple class that stores key Strings as char[]'s in a
@ -36,19 +34,6 @@ import org.apache.lucene.util.Version;
* etc. It is designed to be quick to retrieve items
* by char[] keys without the necessity of converting
* to a String first.
*
* <a name="version"></a>
* <p>You must specify the required {@link Version}
* compatibility when creating {@link CharArrayMap}:
* <ul>
* <li> As of 3.1, supplementary characters are
* properly lowercased.</li>
* </ul>
* Before 3.1 supplementary characters could not be
* lowercased correctly due to the lack of Unicode 4
* support in JDK 1.4. To use instances of
* {@link CharArrayMap} with the behavior before Lucene
* 3.1 pass a {@link Version} &lt; 3.1 to the constructors.
*/
public class CharArrayMap<V> extends AbstractMap<Object,V> {
// private only because missing generics
@ -58,16 +43,12 @@ public class CharArrayMap<V> extends AbstractMap<Object,V> {
private final CharacterUtils charUtils;
private boolean ignoreCase;
private int count;
final Version matchVersion; // package private because used in CharArraySet
char[][] keys; // package private because used in CharArraySet's non Set-conform CharArraySetIterator
V[] values; // package private because used in CharArraySet's non Set-conform CharArraySetIterator
/**
* Create map with enough capacity to hold startSize terms
*
* @param matchVersion
* compatibility match version see <a href="#version">Version
* note</a> above for details.
*
* @param startSize
* the initial capacity
* @param ignoreCase
@ -75,31 +56,27 @@ public class CharArrayMap<V> extends AbstractMap<Object,V> {
* otherwise <code>true</code>.
*/
@SuppressWarnings("unchecked")
public CharArrayMap(Version matchVersion, int startSize, boolean ignoreCase) {
public CharArrayMap(int startSize, boolean ignoreCase) {
this.ignoreCase = ignoreCase;
int size = INIT_SIZE;
while(startSize + (startSize>>2) > size)
size <<= 1;
keys = new char[size][];
values = (V[]) new Object[size];
this.charUtils = CharacterUtils.getInstance(matchVersion);
this.matchVersion = matchVersion;
this.charUtils = CharacterUtils.getInstance();
}
/**
* Creates a map from the mappings in another map.
*
* @param matchVersion
* compatibility match version see <a href="#version">Version
* note</a> above for details.
*
* @param c
* a map whose mappings to be copied
* @param ignoreCase
* <code>false</code> if and only if the set should be case sensitive
* otherwise <code>true</code>.
*/
public CharArrayMap(Version matchVersion, Map<?,? extends V> c, boolean ignoreCase) {
this(matchVersion, c.size(), ignoreCase);
public CharArrayMap(Map<?,? extends V> c, boolean ignoreCase) {
this(c.size(), ignoreCase);
putAll(c);
}
@ -110,7 +87,6 @@ public class CharArrayMap<V> extends AbstractMap<Object,V> {
this.ignoreCase = toCopy.ignoreCase;
this.count = toCopy.count;
this.charUtils = toCopy.charUtils;
this.matchVersion = toCopy.matchVersion;
}
/** Clears all entries in this map. This method is supported for reusing, but not {@link Map#remove}. */
@ -565,18 +541,7 @@ public class CharArrayMap<V> extends AbstractMap<Object,V> {
/**
* Returns a copy of the given map as a {@link CharArrayMap}. If the given map
* is a {@link CharArrayMap} the ignoreCase property will be preserved.
* <p>
* <b>Note:</b> If you intend to create a copy of another {@link CharArrayMap} where
* the {@link Version} of the source map differs from its copy
* {@link #CharArrayMap(Version, Map, boolean)} should be used instead.
* The {@link #copy(Version, Map)} will preserve the {@link Version} of the
* source map it is an instance of {@link CharArrayMap}.
* </p>
*
* @param matchVersion
* compatibility match version see <a href="#version">Version
* note</a> above for details. This argument will be ignored if the
* given map is a {@link CharArrayMap}.
* @param map
* a map to copy
* @return a copy of the given map as a {@link CharArrayMap}. If the given map
@ -584,7 +549,7 @@ public class CharArrayMap<V> extends AbstractMap<Object,V> {
* matchVersion will be of the given map will be preserved.
*/
@SuppressWarnings("unchecked")
public static <V> CharArrayMap<V> copy(final Version matchVersion, final Map<?,? extends V> map) {
public static <V> CharArrayMap<V> copy(final Map<?,? extends V> map) {
if(map == EMPTY_MAP)
return emptyMap();
if(map instanceof CharArrayMap) {
@ -600,7 +565,7 @@ public class CharArrayMap<V> extends AbstractMap<Object,V> {
m.values = values;
return m;
}
return new CharArrayMap<>(matchVersion, map, false);
return new CharArrayMap<>(map, false);
}
/** Returns an empty, unmodifiable map. */
@ -659,7 +624,7 @@ public class CharArrayMap<V> extends AbstractMap<Object,V> {
*/
private static final class EmptyCharArrayMap<V> extends UnmodifiableCharArrayMap<V> {
EmptyCharArrayMap() {
super(new CharArrayMap<V>(Version.LUCENE_CURRENT, 0, false));
super(new CharArrayMap<V>(0, false));
}
@Override

46
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArraySet.java
View File

@ -22,9 +22,6 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.Set;
import org.apache.lucene.util.Version;
/**
* A simple class that stores Strings as char[]'s in a
* hash table. Note that this is not a general purpose
@ -34,18 +31,6 @@ import org.apache.lucene.util.Version;
* is in the set without the necessity of converting it
* to a String first.
*
* <a name="version"></a>
* <p>You must specify the required {@link Version}
* compatibility when creating {@link CharArraySet}:
* <ul>
* <li> As of 3.1, supplementary characters are
* properly lowercased.</li>
* </ul>
* Before 3.1 supplementary characters could not be
* lowercased correctly due to the lack of Unicode 4
* support in JDK 1.4. To use instances of
* {@link CharArraySet} with the behavior before Lucene
* 3.1 pass a {@link Version} < 3.1 to the constructors.
* <P>
* <em>Please note:</em> This class implements {@link java.util.Set Set} but
* does not behave like it should in all cases. The generic type is
@ -64,33 +49,27 @@ public class CharArraySet extends AbstractSet<Object> {
/**
* Create set with enough capacity to hold startSize terms
*
* @param matchVersion
* compatibility match version see <a href="#version">Version
* note</a> above for details.
* @param startSize
* the initial capacity
* @param ignoreCase
* <code>false</code> if and only if the set should be case sensitive
* otherwise <code>true</code>.
*/
public CharArraySet(Version matchVersion, int startSize, boolean ignoreCase) {
this(new CharArrayMap<>(matchVersion, startSize, ignoreCase));
public CharArraySet(int startSize, boolean ignoreCase) {
this(new CharArrayMap<>(startSize, ignoreCase));
}
/**
* Creates a set from a Collection of objects.
*
* @param matchVersion
* compatibility match version see <a href="#version">Version
* note</a> above for details.
* @param c
* a collection whose elements to be placed into the set
* @param ignoreCase
* <code>false</code> if and only if the set should be case sensitive
* otherwise <code>true</code>.
*/
public CharArraySet(Version matchVersion, Collection<?> c, boolean ignoreCase) {
this(matchVersion, c.size(), ignoreCase);
public CharArraySet(Collection<?> c, boolean ignoreCase) {
this(c.size(), ignoreCase);
addAll(c);
}
@ -172,32 +151,21 @@ public class CharArraySet extends AbstractSet<Object> {
/**
* Returns a copy of the given set as a {@link CharArraySet}. If the given set
* is a {@link CharArraySet} the ignoreCase property will be preserved.
* <p>
* <b>Note:</b> If you intend to create a copy of another {@link CharArraySet} where
* the {@link Version} of the source set differs from its copy
* {@link #CharArraySet(Version, Collection, boolean)} should be used instead.
* The {@link #copy(Version, Set)} will preserve the {@link Version} of the
* source set it is an instance of {@link CharArraySet}.
* </p>
*
* @param matchVersion
* compatibility match version see <a href="#version">Version
* note</a> above for details. This argument will be ignored if the
* given set is a {@link CharArraySet}.
* @param set
* a set to copy
* @return a copy of the given set as a {@link CharArraySet}. If the given set
* is a {@link CharArraySet} the ignoreCase property as well as the
* matchVersion will be of the given set will be preserved.
*/
public static CharArraySet copy(final Version matchVersion, final Set<?> set) {
public static CharArraySet copy(final Set<?> set) {
if(set == EMPTY_SET)
return EMPTY_SET;
if(set instanceof CharArraySet) {
final CharArraySet source = (CharArraySet) set;
return new CharArraySet(CharArrayMap.copy(source.map.matchVersion, source.map));
return new CharArraySet(CharArrayMap.copy(source.map));
}
return new CharArraySet(matchVersion, set, false);
return new CharArraySet(set, false);
}
/**

16
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
View File

@ -18,15 +18,12 @@ package org.apache.lucene.analysis.util;
*/
import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.analysis.util.CharacterUtils;
import org.apache.lucene.util.Version;
import org.apache.lucene.analysis.util.CharacterUtils.CharacterBuffer;
/**
@ -36,25 +33,18 @@ public abstract class CharTokenizer extends Tokenizer {
/**
* Creates a new {@link CharTokenizer} instance
*
* @param matchVersion
* Lucene version to match
*/
public CharTokenizer(Version matchVersion) {
charUtils = CharacterUtils.getInstance(matchVersion);
public CharTokenizer() {
}
/**
* Creates a new {@link CharTokenizer} instance
*
* @param matchVersion
* Lucene version to match
* @param factory
* the attribute factory to use for this {@link Tokenizer}
*/
public CharTokenizer(Version matchVersion, AttributeFactory factory) {
public CharTokenizer(AttributeFactory factory) {
super(factory);
charUtils = CharacterUtils.getInstance(matchVersion);
}
private int offset = 0, bufferIndex = 0, dataLen = 0, finalOffset = 0;
@ -64,7 +54,7 @@ public abstract class CharTokenizer extends Tokenizer {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final CharacterUtils charUtils;
private final CharacterUtils charUtils = CharacterUtils.getInstance();
private final CharacterBuffer ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE);
/**

27
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharacterUtils.java
View File

@ -34,29 +34,25 @@ public abstract class CharacterUtils {
private static final Java5CharacterUtils JAVA_5 = new Java5CharacterUtils();
/**
* Returns a {@link CharacterUtils} implementation according to the given
* {@link Version} instance.
*
* @param matchVersion
* a version instance
* Returns a {@link CharacterUtils} implementation.
* @return a {@link CharacterUtils} implementation according to the given
* {@link Version} instance.
*/
public static CharacterUtils getInstance(final Version matchVersion) {
public static CharacterUtils getInstance() {
return JAVA_5;
}
/** explicitly returns a version matching java 4 semantics */
/**
* explicitly returns a version matching java 4 semantics
* @deprecated Only for n-gram backwards compat
*/
@Deprecated
public static CharacterUtils getJava4Instance() {
return JAVA_4;
}
/**
* Returns the code point at the given index of the {@link CharSequence}.
* Depending on the {@link Version} passed to
* {@link CharacterUtils#getInstance(Version)} this method mimics the behavior
* of {@link Character#codePointAt(char[], int)} as it would have been
* available on a Java 1.4 JVM or on a later virtual machine version.
*
* @param seq
* a character sequence
@ -75,10 +71,6 @@ public abstract class CharacterUtils {
/**
* Returns the code point at the given index of the char array where only elements
* with index less than the limit are used.
* Depending on the {@link Version} passed to
* {@link CharacterUtils#getInstance(Version)} this method mimics the behavior
* of {@link Character#codePointAt(char[], int)} as it would have been
* available on a Java 1.4 JVM or on a later virtual machine version.
*
* @param chars
* a character array
@ -188,10 +180,7 @@ public abstract class CharacterUtils {
* the middle of a surrogate pair, even if there are remaining characters in
* the {@link Reader}.
* <p>
* Depending on the {@link Version} passed to
* {@link CharacterUtils#getInstance(Version)} this method implements
* supplementary character awareness when filling the given buffer. For all
* {@link Version} &gt; 3.0 {@link #fill(CharacterBuffer, Reader, int)} guarantees
* This method guarantees
* that the given {@link CharacterBuffer} will never contain a high surrogate
* character as the last element in the buffer unless it is the last available
* character in the reader. In other words, high and low surrogate pairs will

Some files were not shown because too many files have changed in this diff Show More