diff --git a/pom.xml b/pom.xml index 8fa4d30e91e..d759a86378b 100644 --- a/pom.xml +++ b/pom.xml @@ -114,13 +114,6 @@ compile - - commons-codec - commons-codec - 1.6 - compile - - joda-time joda-time @@ -312,7 +305,6 @@ net.sf.trove4j:trove4j org.elasticsearch:es-jsr166y org.elasticsearch:es-jsr166e - commons-codec:commons-codec org.mvel:mvel2 org.codehaus.jackson:jackson-core-asl org.codehaus.jackson:jackson-smile @@ -338,10 +330,6 @@ jsr166e org.elasticsearch.common.util.concurrent.jsr166e - - org.apache.commons.codec - org.elasticsearch.common.codec - org.mvel2 org.elasticsearch.common.mvel2 diff --git a/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java b/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java index 7ce840d9cb3..db8c9bf2fb7 100644 --- a/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java +++ b/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.NoClassSettingsException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory; import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory; -import org.elasticsearch.index.analysis.phonetic.PhoneticTokenFilterFactory; import org.elasticsearch.indices.analysis.IndicesAnalysisService; import java.util.LinkedList; @@ -452,7 +451,6 @@ public class AnalysisModule extends AbstractModule { tokenFiltersBindings.processTokenFilter("elision", ElisionTokenFilterFactory.class); tokenFiltersBindings.processTokenFilter("pattern_replace", PatternReplaceTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("phonetic", PhoneticTokenFilterFactory.class); tokenFiltersBindings.processTokenFilter("dictionary_decompounder", DictionaryCompoundWordTokenFilterFactory.class); tokenFiltersBindings.processTokenFilter("hyphenation_decompounder", HyphenationCompoundWordTokenFilterFactory.class); diff --git a/src/main/java/org/elasticsearch/index/analysis/phonetic/DoubleMetaphoneFilter.java b/src/main/java/org/elasticsearch/index/analysis/phonetic/DoubleMetaphoneFilter.java deleted file mode 100644 index bd73e509ef9..00000000000 --- a/src/main/java/org/elasticsearch/index/analysis/phonetic/DoubleMetaphoneFilter.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.analysis.phonetic; - -import org.apache.commons.codec.language.DoubleMetaphone; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; - -import java.io.IOException; -import java.util.LinkedList; - -public final class DoubleMetaphoneFilter extends TokenFilter { - - private static final String TOKEN_TYPE = "DoubleMetaphone"; - - private final LinkedList remainingTokens = new LinkedList(); - private final DoubleMetaphone encoder; - private final boolean inject; - private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); - private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class); - - public DoubleMetaphoneFilter(TokenStream input, DoubleMetaphone encoder, boolean inject) { - super(input); - this.encoder = encoder; - this.inject = inject; - } - - @Override - public boolean incrementToken() throws IOException { - for (; ; ) { - - if (!remainingTokens.isEmpty()) { - // clearAttributes(); // not currently necessary - restoreState(remainingTokens.removeFirst()); - return true; - } - - if (!input.incrementToken()) return false; - - int len = termAtt.length(); - if (len == 0) return true; // pass through zero length terms - - int firstAlternativeIncrement = inject ? 0 : posAtt.getPositionIncrement(); - - String v = termAtt.toString(); - String primaryPhoneticValue = encoder.doubleMetaphone(v); - String alternatePhoneticValue = encoder.doubleMetaphone(v, true); - - // a flag to lazily save state if needed... this avoids a save/restore when only - // one token will be generated. - boolean saveState = inject; - - if (primaryPhoneticValue != null && primaryPhoneticValue.length() > 0 && !primaryPhoneticValue.equals(v)) { - if (saveState) { - remainingTokens.addLast(captureState()); - } - posAtt.setPositionIncrement(firstAlternativeIncrement); - firstAlternativeIncrement = 0; - termAtt.setEmpty().append(primaryPhoneticValue); - saveState = true; - } - - if (alternatePhoneticValue != null && alternatePhoneticValue.length() > 0 - && !alternatePhoneticValue.equals(primaryPhoneticValue) - && !primaryPhoneticValue.equals(v)) { - if (saveState) { - remainingTokens.addLast(captureState()); - saveState = false; - } - posAtt.setPositionIncrement(firstAlternativeIncrement); - termAtt.setEmpty().append(alternatePhoneticValue); - saveState = true; - } - - // Just one token to return, so no need to capture/restore - // any state, simply return it. - if (remainingTokens.isEmpty()) { - return true; - } - - if (saveState) { - remainingTokens.addLast(captureState()); - } - } - } - - @Override - public void reset() throws IOException { - input.reset(); - remainingTokens.clear(); - } -} diff --git a/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticFilter.java b/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticFilter.java deleted file mode 100644 index d98c66cd11b..00000000000 --- a/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticFilter.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.analysis.phonetic; - -import org.apache.commons.codec.Encoder; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; - -import java.io.IOException; - -/** - * Create tokens for phonetic matches. See: - * http://jakarta.apache.org/commons/codec/api-release/org/apache/commons/codec/language/package-summary.html - */ -// LUCENE MONITOR -public class PhoneticFilter extends TokenFilter { - - protected boolean inject = true; - protected Encoder encoder = null; - protected String name = null; - - protected State save = null; - private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); - private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class); - - public PhoneticFilter(TokenStream in, Encoder encoder, String name, boolean inject) { - super(in); - this.encoder = encoder; - this.name = name; - this.inject = inject; - } - - @Override - public boolean incrementToken() throws IOException { - if (save != null) { - // clearAttributes(); // not currently necessary - restoreState(save); - save = null; - return true; - } - - if (!input.incrementToken()) return false; - - // pass through zero-length terms - if (termAtt.length() == 0) return true; - - String value = termAtt.toString(); - String phonetic = null; - try { - String v = encoder.encode(value).toString(); - if (v.length() > 0 && !value.equals(v)) phonetic = v; - } catch (Exception ignored) { - } // just use the direct text - - if (phonetic == null) return true; - - if (!inject) { - // just modify this token - termAtt.setEmpty().append(phonetic); - return true; - } - - // We need to return both the original and the phonetic tokens. - // to avoid a orig=captureState() change_to_phonetic() saved=captureState() restoreState(orig) - // we return the phonetic alternative first - - int origOffset = posAtt.getPositionIncrement(); - posAtt.setPositionIncrement(0); - save = captureState(); - - posAtt.setPositionIncrement(origOffset); - termAtt.setEmpty().append(phonetic); - return true; - } - - @Override - public void reset() throws IOException { - input.reset(); - save = null; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java b/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java deleted file mode 100644 index b90671f5519..00000000000 --- a/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.analysis.phonetic; - -import org.apache.commons.codec.Encoder; -import org.apache.commons.codec.language.*; -import org.apache.commons.codec.language.bm.BeiderMorseEncoder; -import org.apache.commons.codec.language.bm.NameType; -import org.apache.commons.codec.language.bm.RuleType; -import org.apache.lucene.analysis.TokenStream; -import org.elasticsearch.ElasticSearchIllegalArgumentException; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.AnalysisSettingsRequired; -import org.elasticsearch.index.settings.IndexSettings; - -/** - * - */ -@AnalysisSettingsRequired -public class PhoneticTokenFilterFactory extends AbstractTokenFilterFactory { - - private final Encoder encoder; - - private final boolean replace; - - @Inject - public PhoneticTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); - this.replace = settings.getAsBoolean("replace", true); - String encoder = settings.get("encoder"); - if (encoder == null) { - throw new ElasticSearchIllegalArgumentException("encoder must be set on phonetic token filter"); - } - if ("metaphone".equalsIgnoreCase(encoder)) { - this.encoder = new Metaphone(); - } else if ("soundex".equalsIgnoreCase(encoder)) { - this.encoder = new Soundex(); - } else if ("caverphone1".equalsIgnoreCase(encoder)) { - this.encoder = new Caverphone1(); - } else if ("caverphone2".equalsIgnoreCase(encoder)) { - this.encoder = new Caverphone2(); - } else if ("caverphone".equalsIgnoreCase(encoder)) { - this.encoder = new Caverphone2(); - } else if ("refined_soundex".equalsIgnoreCase(encoder) || "refinedSoundex".equalsIgnoreCase(encoder)) { - this.encoder = new RefinedSoundex(); - } else if ("cologne".equalsIgnoreCase(encoder)) { - this.encoder = new ColognePhonetic(); - } else if ("double_metaphone".equalsIgnoreCase(encoder) || "doubleMetaphone".equalsIgnoreCase(encoder)) { - DoubleMetaphone doubleMetaphone = new DoubleMetaphone(); - doubleMetaphone.setMaxCodeLen(settings.getAsInt("max_code_len", doubleMetaphone.getMaxCodeLen())); - this.encoder = doubleMetaphone; - } else if ("bm".equalsIgnoreCase(encoder) || "beider_morse".equalsIgnoreCase(encoder)) { - BeiderMorseEncoder bm = new BeiderMorseEncoder(); - String ruleType = settings.get("rule_type", "approx"); - if ("approx".equalsIgnoreCase(ruleType)) { - bm.setRuleType(RuleType.APPROX); - } else if ("exact".equalsIgnoreCase(ruleType)) { - bm.setRuleType(RuleType.EXACT); - } else { - throw new ElasticSearchIllegalArgumentException("No matching rule type [" + ruleType + "] for beider morse encoder"); - } - String nameType = settings.get("name_type", "generic"); - if ("GENERIC".equalsIgnoreCase(nameType)) { - bm.setNameType(NameType.GENERIC); - } else if ("ASHKENAZI".equalsIgnoreCase(nameType)) { - bm.setNameType(NameType.ASHKENAZI); - } else if ("SEPHARDIC".equalsIgnoreCase(nameType)) { - bm.setNameType(NameType.SEPHARDIC); - } - this.encoder = bm; - } else { - throw new ElasticSearchIllegalArgumentException("unknown encoder [" + encoder + "] for phonetic token filter"); - } - } - - @Override - public TokenStream create(TokenStream tokenStream) { - if (encoder instanceof DoubleMetaphone) { - return new DoubleMetaphoneFilter(tokenStream, (DoubleMetaphone) encoder, !replace); - } - return new PhoneticFilter(tokenStream, encoder, name(), !replace); - } -} \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java b/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java index a74d39f5825..165d3748c7f 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java @@ -32,11 +32,10 @@ import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNameModule; import org.elasticsearch.index.analysis.*; -import org.elasticsearch.test.unit.index.analysis.filter1.MyFilterTokenFilterFactory; -import org.elasticsearch.index.analysis.phonetic.PhoneticTokenFilterFactory; import org.elasticsearch.index.settings.IndexSettingsModule; import org.elasticsearch.indices.analysis.IndicesAnalysisModule; import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.test.unit.index.analysis.filter1.MyFilterTokenFilterFactory; import org.hamcrest.MatcherAssert; import org.testng.annotations.Test; @@ -109,12 +108,6 @@ public class AnalysisModuleTests { analyzer = analysisService.analyzer("alias1").analyzer(); assertThat(analyzer, instanceOf(StandardAnalyzer.class)); - // check phonetic - analyzer = analysisService.analyzer("custom3").analyzer(); - assertThat(analyzer, instanceOf(CustomAnalyzer.class)); - CustomAnalyzer custom3 = (CustomAnalyzer) analyzer; - assertThat(custom3.tokenFilters()[0], instanceOf(PhoneticTokenFilterFactory.class)); - // check custom class name (my) analyzer = analysisService.analyzer("custom4").analyzer(); assertThat(analyzer, instanceOf(CustomAnalyzer.class)); diff --git a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json index 6b7a8ec67d7..b674552fe2c 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json +++ b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json @@ -53,10 +53,6 @@ "tokenizer":"standard", "char_filter":["html_strip", "my_html"] }, - "custom3":{ - "tokenizer":"standard", - "filter":["metaphone"] - }, "custom4":{ "tokenizer":"standard", "filter":["my"] diff --git a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml index f6c1bc0b0b7..3f4a5c29f38 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml +++ b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml @@ -38,9 +38,6 @@ index : custom2 : tokenizer : standard char_filter : [html_strip, my_html] - custom3 : - tokenizer : standard - filter : [metaphone] custom4 : tokenizer : standard filter : [my]