Warn users with IAE if pos_inc is disabled in 'stop' filter

src/main/java/org/elasticsearch/index/analysis/StopTokenFilterFactory.java

@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.core.StopAnalyzer;
 import org.apache.lucene.analysis.core.StopFilter;
 import org.apache.lucene.analysis.util.CharArraySet;
 import org.apache.lucene.util.Version;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.assistedinject.Assisted;
 import org.elasticsearch.common.settings.Settings;
@@ -50,7 +51,11 @@ public class StopTokenFilterFactory extends AbstractTokenFilterFactory {
         super(index, indexSettings, name, settings);
         this.ignoreCase = settings.getAsBoolean("ignore_case", false);
         this.stopWords = Analysis.parseStopWords(env, settings, StopAnalyzer.ENGLISH_STOP_WORDS_SET, version, ignoreCase);
-        this.enablePositionIncrements = settings.getAsBoolean("enable_position_increments", version.onOrAfter(Version.LUCENE_30));
+        this.enablePositionIncrements = settings.getAsBoolean("enable_position_increments", true);
+        if (!enablePositionIncrements && version.onOrAfter(Version.LUCENE_44)) {
+            throw new ElasticSearchIllegalArgumentException("[enable_position_increments: false] is not supported anymore as of Lucene 4.4 as it can create broken token streams."
+                    + " Please fix your analysis chain or use an older compatibility version (<=4.3) but beware that it might cause unexpected behavior.");
+        }
     }
 
     @Override

src/test/java/org/elasticsearch/index/analysis/StopTokenFilterTests.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.util.Version;
+import org.elasticsearch.common.inject.ProvisionException;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.ImmutableSettings.Builder;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ElasticSearchTokenStreamTestCase;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+
+public class StopTokenFilterTests extends ElasticSearchTokenStreamTestCase {
+
+    @Test(expected = ProvisionException.class)
+    public void testPositionIncrementSetting() throws IOException {
+        Builder builder = ImmutableSettings.settingsBuilder().put("index.analysis.filter.my_stop.type", "stop")
+                .put("index.analysis.filter.my_stop.enable_position_increments", false);
+        if (random().nextBoolean()) {
+            builder.put("index.analysis.filter.my_stop.version", "4.4");
+        }
+        Settings settings = builder.build();
+        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+        analysisService.tokenFilter("my_stop");
+    }
+
+    @Test
+    public void testCorrectPositionIncrementSetting() throws IOException {
+        Builder builder = ImmutableSettings.settingsBuilder().put("index.analysis.filter.my_stop.type", "stop");
+        if (random().nextBoolean()) {
+            builder.put("index.analysis.filter.my_stop.enable_position_increments", true);
+        }
+        if (random().nextBoolean()) {
+            builder.put("index.analysis.filter.my_stop.version", Version.values()[random().nextInt(Version.values().length)]);
+        }
+        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(builder.build());
+        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_stop");
+        assertThat(tokenFilter, instanceOf(StopTokenFilterFactory.class));
+        TokenStream create = tokenFilter.create(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("foo bar")));
+        assertThat(create, instanceOf(StopFilter.class));
+        assertThat(((StopFilter)create).getEnablePositionIncrements(), equalTo(true));
+
+    }
+
+    @Test
+    public void testDeprectedPositionIncrementSettingWithVerions() throws IOException {
+        Settings settings = ImmutableSettings.settingsBuilder().put("index.analysis.filter.my_stop.type", "stop")
+                .put("index.analysis.filter.my_stop.enable_position_increments", false).put("index.analysis.filter.my_stop.version", "4.3")
+                .build();
+        AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+        TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_stop");
+        assertThat(tokenFilter, instanceOf(StopTokenFilterFactory.class));
+        TokenStream create = tokenFilter.create(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("foo bar")));
+        assertThat(create, instanceOf(StopFilter.class));
+        assertThat(((StopFilter)create).getEnablePositionIncrements(), equalTo(false));
+
+    }
+
+}