Remove `omit_term_freq_and_positions` for new indices

`omit_term_freq_and_positions` was deprecated in `0.20` and
is not documented anymore. We should reject indices that are
created with this option in the future.

Closes #4722

src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java

@@ -21,13 +21,14 @@ package org.elasticsearch.index.mapper;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.compress.CompressedString;
 import org.elasticsearch.common.geo.ShapesAvailability;
-import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -64,16 +65,11 @@ public class DocumentMapperParser extends AbstractIndexComponent {
     private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser();
 
     private final Object typeParsersMutex = new Object();
+    private final Version indexVersionCreated;
 
     private volatile ImmutableMap<String, Mapper.TypeParser> typeParsers;
     private volatile ImmutableMap<String, Mapper.TypeParser> rootTypeParsers;
 
-    public DocumentMapperParser(Index index, AnalysisService analysisService, PostingsFormatService postingsFormatService,
-                                DocValuesFormatService docValuesFormatService, SimilarityLookupService similarityLookupService) {
-        this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, analysisService, postingsFormatService, docValuesFormatService,
-                similarityLookupService);
-    }
-
     public DocumentMapperParser(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService,
                                 PostingsFormatService postingsFormatService, DocValuesFormatService docValuesFormatService,
                                 SimilarityLookupService similarityLookupService) {
@@ -123,6 +119,9 @@ public class DocumentMapperParser extends AbstractIndexComponent {
                 .put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser())
                 .put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser())
                 .immutableMap();
+        assert indexSettings.get(IndexMetaData.SETTING_UUID) == null // if the UUDI is there the index has actually been created otherwise this might be a test
+                || indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) != null : IndexMetaData.SETTING_VERSION_CREATED + " not set in IndexSettings";
+        indexVersionCreated = indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
     }
 
     public void putTypeParser(String type, Mapper.TypeParser typeParser) {
@@ -142,7 +141,7 @@ public class DocumentMapperParser extends AbstractIndexComponent {
     }
 
     public Mapper.TypeParser.ParserContext parserContext() {
-        return new Mapper.TypeParser.ParserContext(postingsFormatService, docValuesFormatService, analysisService, similarityLookupService, typeParsers);
+        return new Mapper.TypeParser.ParserContext(postingsFormatService, docValuesFormatService, analysisService, similarityLookupService, typeParsers, indexVersionCreated);
     }
 
     public DocumentMapper parse(String source) throws MapperParsingException {
@@ -199,8 +198,8 @@ public class DocumentMapperParser extends AbstractIndexComponent {
             }
         }
 
-        Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(postingsFormatService, docValuesFormatService, analysisService, similarityLookupService, typeParsers);
 
+        Mapper.TypeParser.ParserContext parserContext = parserContext();
         DocumentMapper.Builder docBuilder = doc(index.name(), indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext));
 
         for (Map.Entry<String, Object> entry : mapping.entrySet()) {

src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -99,14 +99,17 @@ public interface Mapper extends ToXContent {
 
             private final ImmutableMap<String, TypeParser> typeParsers;
 
+            private final Version indexVersionCreated;
+
             public ParserContext(PostingsFormatService postingsFormatService, DocValuesFormatService docValuesFormatService,
                                  AnalysisService analysisService, SimilarityLookupService similarityLookupService,
-                                 ImmutableMap<String, TypeParser> typeParsers) {
+                                 ImmutableMap<String, TypeParser> typeParsers, Version indexVersionCreated) {
                 this.postingsFormatService = postingsFormatService;
                 this.docValuesFormatService = docValuesFormatService;
                 this.analysisService = analysisService;
                 this.similarityLookupService = similarityLookupService;
                 this.typeParsers = typeParsers;
+                this.indexVersionCreated = indexVersionCreated;
             }
 
             public AnalysisService analysisService() {
@@ -128,6 +131,10 @@ public interface Mapper extends ToXContent {
             public TypeParser typeParser(String type) {
                 return typeParsers.get(Strings.toUnderscoreCase(type));
             }
+
+            public Version indexVersionCreated() {
+                return indexVersionCreated;
+            }
         }
 
         Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException;

src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java

@@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.core;
 
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
@@ -198,8 +199,12 @@ public class TypeParsers {
                     }
                 }
             } else if (propName.equals("omit_term_freq_and_positions")) {
+                final IndexOptions op = nodeBooleanValue(propNode) ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+                if (parserContext.indexVersionCreated().onOrAfter(Version.V_1_0_0)) {
+                    throw new ElasticsearchParseException("'omit_term_freq_and_positions' is not supported anymore - use ['index_options' : '" + op.name() + "']  instead");
+                }
                 // deprecated option for BW compat
-                builder.indexOptions(nodeBooleanValue(propNode) ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+                builder.indexOptions(op);
             } else if (propName.equals("index_options")) {
                 builder.indexOptions(nodeIndexOptionValue(propNode));
             } else if (propName.equals("analyzer")) {

src/test/java/org/elasticsearch/count/query/SimpleQueryTests.java

@@ -130,25 +130,6 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
         assertHitCount(countResponse, 3l);
     }
 
-    @Test
-    public void testOmitTermFreqsAndPositions() throws Exception {
-        // backwards compat test!
-        assertAcked(prepareCreate("test")
-                .addMapping("type1", "field1", "type=string,omit_term_freq_and_positions=true")
-                .setSettings(SETTING_NUMBER_OF_SHARDS, 1));
-
-        indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "quick brown fox", "field2", "quick brown fox"),
-                client().prepareIndex("test", "type1", "2").setSource("field1", "quick lazy huge brown fox", "field2", "quick lazy huge brown fox"));
-
-        CountResponse countResponse = client().prepareCount().setQuery(QueryBuilders.matchQuery("field2", "quick brown").type(Type.PHRASE).slop(0)).get();
-        assertHitCount(countResponse, 1l);
-        try {
-            client().prepareCount().setQuery(QueryBuilders.matchQuery("field1", "quick brown").type(Type.PHRASE).slop(0)).get();
-        } catch (SearchPhaseExecutionException e) {
-            assertTrue(e.getMessage().endsWith("IllegalStateException[field \"field1\" was indexed without position data; cannot run PhraseQuery (term=quick)]; }"));
-        }
-    }
-
     @Test
     public void queryStringAnalyzedWildcard() throws Exception {
         assertAcked(prepareCreate("test").setSettings(SETTING_NUMBER_OF_SHARDS, 1));

src/test/java/org/elasticsearch/index/mapper/MapperTestUtils.java

@@ -45,7 +45,7 @@ import org.elasticsearch.indices.fielddata.breaker.DummyCircuitBreakerService;
 public class MapperTestUtils {
 
     public static DocumentMapperParser newParser() {
-        return new DocumentMapperParser(new Index("test"), newAnalysisService(), new PostingsFormatService(new Index("test")),
+        return new DocumentMapperParser(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, newAnalysisService(), new PostingsFormatService(new Index("test")),
                 new DocValuesFormatService(new Index("test")), newSimilarityLookupService());
     }
 

src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java

@@ -21,13 +21,16 @@ package org.elasticsearch.search.query;
 
 import org.apache.lucene.util.English;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.*;
 import org.elasticsearch.index.query.CommonTermsQueryBuilder.Operator;
 import org.elasticsearch.index.query.MatchQueryBuilder.Type;
@@ -325,21 +328,32 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
 
     @Test
     public void testOmitTermFreqsAndPositions() throws Exception {
-        // backwards compat test!
+        Version version = Version.CURRENT;
-        assertAcked(client().admin().indices().prepareCreate("test")
+        int iters = atLeast(10);
-                .addMapping("type1", "field1", "type=string,omit_term_freq_and_positions=true")
+        for (int i = 0; i < iters; i++) {
-                .setSettings(SETTING_NUMBER_OF_SHARDS, 1));
+            try {
+                // backwards compat test!
+                assertAcked(client().admin().indices().prepareCreate("test")
+                        .addMapping("type1", "field1", "type=string,omit_term_freq_and_positions=true")
+                        .setSettings(SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_VERSION_CREATED, version.id));
+                assertThat(version.onOrAfter(Version.V_1_0_0), equalTo(false));
+                indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "quick brown fox", "field2", "quick brown fox"),
+                        client().prepareIndex("test", "type1", "2").setSource("field1", "quick lazy huge brown fox", "field2", "quick lazy huge brown fox"));
 
-        indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "quick brown fox", "field2", "quick brown fox"),
+                SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("field2", "quick brown").type(MatchQueryBuilder.Type.PHRASE).slop(0)).get();
-                client().prepareIndex("test", "type1", "2").setSource("field1", "quick lazy huge brown fox", "field2", "quick lazy huge brown fox"));
+                assertHitCount(searchResponse, 1l);
-
+                try {
-        SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("field2", "quick brown").type(MatchQueryBuilder.Type.PHRASE).slop(0)).get();
+                    client().prepareSearch().setQuery(matchQuery("field1", "quick brown").type(MatchQueryBuilder.Type.PHRASE).slop(0)).get();
-        assertHitCount(searchResponse, 1l);
+                    fail("SearchPhaseExecutionException should have been thrown");
-        try {
+                } catch (SearchPhaseExecutionException e) {
-            client().prepareSearch().setQuery(matchQuery("field1", "quick brown").type(MatchQueryBuilder.Type.PHRASE).slop(0)).get();
+                    assertTrue(e.getMessage().endsWith("IllegalStateException[field \"field1\" was indexed without position data; cannot run PhraseQuery (term=quick)]; }"));
-            fail("SearchPhaseExecutionException should have been thrown");
+                }
-        } catch (SearchPhaseExecutionException e) {
+                wipeIndices("test");
-            assertTrue(e.getMessage().endsWith("IllegalStateException[field \"field1\" was indexed without position data; cannot run PhraseQuery (term=quick)]; }"));
+            } catch (MapperParsingException ex) {
+                assertThat(version.toString(), version.onOrAfter(Version.V_1_0_0), equalTo(true));
+                assertThat(ex.getCause().getMessage(), equalTo("'omit_term_freq_and_positions' is not supported anymore - use ['index_options' : 'DOCS_ONLY']  instead"));
+            }
+            version = randomVersion();
         }
     }