honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#2436 expose KeepWordTokenFilter by default

This commit is contained in:
Simon Willnauer 2012-11-22 21:54:37 +01:00
parent 65a43d3ad4
commit 32a0772821
5 changed files with 228 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java
View File

@ -480,6 +480,7 @@ public class AnalysisModule extends AbstractModule {
tokenFiltersBindings.processTokenFilter("word_delimiter", WordDelimiterTokenFilterFactory.class);
tokenFiltersBindings.processTokenFilter("synonym", SynonymTokenFilterFactory.class);
tokenFiltersBindings.processTokenFilter("elision", ElisionTokenFilterFactory.class);
tokenFiltersBindings.processTokenFilter("keep", KeepWordFilterFactory.class);
tokenFiltersBindings.processTokenFilter("pattern_replace", PatternReplaceTokenFilterFactory.class);
tokenFiltersBindings.processTokenFilter("dictionary_decompounder", DictionaryCompoundWordTokenFilterFactory.class);

97
src/main/java/org/elasticsearch/index/analysis/KeepWordFilterFactory.java Normal file
View File

@ -0,0 +1,97 @@
package org.elasticsearch.index.analysis;
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import java.util.Arrays;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.KeepWordFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.analysis.IndicesAnalysisService;
/**
* A {@link TokenFilterFactory} for {@link KeepWordFilter}. This filter only
* keep tokens that are contained in the term set configured via
* {@value #KEEP_WORDS_KEY} setting. This filter acts like an inverse stop
* filter.
*
* Configuration options:
*
* <ul>
* <li>{@value #KEEP_WORDS_KEY} the array of words / tokens to keep.</li>
*
* <li>{@value #KEEP_WORDS_PATH_KEY} an reference to a file containing the words
* / tokens to keep. Note: this is an alternative to {@value #KEEP_WORDS_KEY} if
* both are set an exception will be thrown.</li>
*
* <li>{@value #ENABLE_POS_INC_KEY} <code>true</code> iff the filter should
* maintain position increments for dropped tokens. The default is
* <code>true</code>.</li>
*
* <li>{@value #KEEP_WORDS_CASE_KEY} to use case sensitive keep words. The
* default is <code>false</code> which corresponds to case-sensitive.</li>
* </ul>
*
* @see StopTokenFilterFactory
*
*/
@AnalysisSettingsRequired
public class KeepWordFilterFactory extends AbstractTokenFilterFactory {
private Boolean enablePositionIncrements;
private CharArraySet keepWords;
private static final String KEEP_WORDS_KEY = "keep_words";
private static final String KEEP_WORDS_PATH_KEY = KEEP_WORDS_KEY + "_path";
private static final String KEEP_WORDS_CASE_KEY = KEEP_WORDS_KEY + "_case"; // for javadoc
private static final String ENABLE_POS_INC_KEY = "enable_position_increments";
@Inject
public KeepWordFilterFactory(Index index, @IndexSettings Settings indexSettings,
Environment env, IndicesAnalysisService indicesAnalysisService, Map<String, TokenizerFactoryFactory> tokenizerFactories,
@Assisted String name, @Assisted Settings settings) {
super(index, indexSettings, name, settings);
final String[] arrayKeepWords = settings.getAsArray(KEEP_WORDS_KEY);
final String keepWordsPath = settings.get(KEEP_WORDS_PATH_KEY, null);
if (!(arrayKeepWords == null ^ keepWordsPath == null)) {
// we don't allow both or non
throw new ElasticSearchIllegalArgumentException("keep requires either `" + KEEP_WORDS_KEY + "` or `"
+ KEEP_WORDS_PATH_KEY + "` to be configured");
}
this.enablePositionIncrements = settings.getAsBoolean(ENABLE_POS_INC_KEY, true);
this.keepWords = Analysis.getWordSet(env, settings, KEEP_WORDS_KEY, version);
}
@Override
public TokenStream create(TokenStream tokenStream) {
return new KeepWordFilter(enablePositionIncrements, tokenStream, keepWords);
}
}

21
src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisTestsHelper.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.test.unit.index.analysis;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
@ -45,6 +46,11 @@ public class AnalysisTestsHelper {
Settings settings = ImmutableSettings.settingsBuilder()
.loadFromClasspath(resource).build();
return createAnalysisServiceFromSettings(settings);
}
public static AnalysisService createAnalysisServiceFromSettings(
Settings settings) {
Index index = new Index("test");
Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings),
@ -71,4 +77,19 @@ public class AnalysisTestsHelper {
}
Assert.assertEquals(i, expected.length, "not all tokens produced");
}
public static void assertSimpleTSOutput(TokenStream stream, String[] expected, int[] posInc) throws IOException {
stream.reset();
CharTermAttribute termAttr = stream.getAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncAttr = stream.getAttribute(PositionIncrementAttribute.class);
Assert.assertNotNull(termAttr);
int i = 0;
while (stream.incrementToken()) {
Assert.assertTrue(i < expected.length, "got extra term: " + termAttr.toString());
Assert.assertEquals(termAttr.toString(), expected[i], "expected different term at index " + i);
Assert.assertEquals(posIncAttr.getPositionIncrement(), posInc[i]);
i++;
}
Assert.assertEquals(i, expected.length, "not all tokens produced");
}
}

90
src/test/java/org/elasticsearch/test/unit/index/analysis/KeepFilterFactoryTests.java Normal file
View File

@ -0,0 +1,90 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.unit.index.analysis;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.util.Version;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.analysis.KeepWordFilterFactory;
import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
import java.io.IOException;
import java.io.StringReader;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.instanceOf;
public class KeepFilterFactoryTests {
private static final String RESOURCE = "org/elasticsearch/test/unit/index/analysis/keep_analysis.json";
@Test
public void testLoadWithoutSettings() {
AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(RESOURCE);
TokenFilterFactory tokenFilter = analysisService.tokenFilter("keep");
Assert.assertNull(tokenFilter);
}
@Test
public void testLoadOverConfiguredSettings() {
Settings settings = ImmutableSettings.settingsBuilder()
.put("index.analysis.filter.broken_keep_filter.type", "keep")
.put("index.analysis.filter.broken_keep_filter.keep_words_path", "does/not/exists.txt")
.put("index.analysis.filter.broken_keep_filter.keep_words", "[\"Hello\", \"worlD\"]")
.build();
try {
AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
Assert.fail("path and array are configured");
} catch (Exception e) {
assertThat(e.getCause(), instanceOf(ElasticSearchIllegalArgumentException.class));
}
}
@Test
public void testCaseInsensitiveMapping() throws IOException {
AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(RESOURCE);
TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_keep_filter");
assertThat(tokenFilter, instanceOf(KeepWordFilterFactory.class));
String source = "hello small world";
String[] expected = new String[]{"hello", "world"};
Tokenizer tokenizer = new WhitespaceTokenizer(Version.LUCENE_40, new StringReader(source));
AnalysisTestsHelper.assertSimpleTSOutput(tokenFilter.create(tokenizer), expected, new int[] {1,2});
}
@Test
public void testCaseSensitiveMapping() throws IOException {
AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(RESOURCE);
TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_case_sensitive_keep_filter");
assertThat(tokenFilter, instanceOf(KeepWordFilterFactory.class));
String source = "Hello small world";
String[] expected = new String[]{"Hello"};
Tokenizer tokenizer = new WhitespaceTokenizer(Version.LUCENE_40, new StringReader(source));
AnalysisTestsHelper.assertSimpleTSOutput(tokenFilter.create(tokenizer), expected, new int[] {1});
}
}

19
src/test/java/org/elasticsearch/test/unit/index/analysis/keep_analysis.json Normal file
View File

@ -0,0 +1,19 @@
{
"index":{
"analysis":{
"filter":{
"my_keep_filter":{
"type":"keep",
"keep_words" : ["Hello", "worlD"],
"enable_position_increments" : true,
"keep_words_case" : true
},
"my_case_sensitive_keep_filter":{
"type":"keep",
"keep_words" : ["Hello", "worlD"],
"enable_position_increments" : false
}
}
}
}
}