Add serialization test for FieldMappers when include_defaults=true (#58235) (#58328)

Fixes a bug in TextFieldMapper serialization when index is false, and adds a
base-class test to ensure that all field mappers are tested against all variations
with defaults both included and excluded.

Fixes #58188

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

@@ -72,6 +72,10 @@ public class ScaledFloatFieldMapper extends FieldMapper {
     public static final String CONTENT_TYPE = "scaled_float";
     // use the same default as numbers
     private static final Setting<Boolean> COERCE_SETTING = NumberFieldMapper.COERCE_SETTING;
+    private static final FieldType FIELD_TYPE = new FieldType();
+    static {
+        FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+    }
 
     public static class Builder extends FieldMapper.Builder<Builder> {
 
@@ -82,7 +86,7 @@ public class ScaledFloatFieldMapper extends FieldMapper {
         private Double nullValue;
 
         public Builder(String name) {
-            super(name, new FieldType());
+            super(name, FIELD_TYPE);
             builder = this;
         }
 

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java

@@ -45,7 +45,7 @@ public class RankFeatureFieldMapperTests extends FieldMapperTestCase<RankFeature
 
     @Override
     protected Set<String> unsupportedProperties() {
-        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values");
+        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values", "index");
     }
 
     @Before

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java

@@ -40,7 +40,7 @@ public class RankFeaturesFieldMapperTests extends FieldMapperTestCase<RankFeatur
 
     @Override
     protected Set<String> unsupportedProperties() {
-        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values");
+        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values", "index");
     }
 
     IndexService indexService;

plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

@@ -658,7 +658,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
     @Override
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);
-        if (includeDefaults || (mappedFieldType.isSearchable() && fieldType.indexOptions() != IndexOptions.DOCS)) {
+        if (fieldType.indexOptions() != IndexOptions.NONE && (includeDefaults || fieldType.indexOptions() != IndexOptions.DOCS)) {
             builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
         }
         if (nullValue != null) {

plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java

@@ -58,7 +58,7 @@ public class Murmur3FieldMapperTests extends FieldMapperTestCase<Murmur3FieldMap
 
     @Override
     protected Set<String> unsupportedProperties() {
-        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values");
+        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "index");
     }
 
     @Before

server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java

@@ -148,9 +148,11 @@ public class SimpleGetFieldMappingsIT extends ESIntegTestCase {
     public void testSimpleGetFieldMappingsWithDefaults() throws Exception {
         assertAcked(prepareCreate("test").addMapping("type", getMappingForType("type")));
         client().admin().indices().preparePutMapping("test").setType("type").setSource("num", "type=long").get();
+        client().admin().indices().preparePutMapping("test").setType("type")
+            .setSource("field2", "type=text,index=false").get();
 
         GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings()
-            .setFields("num", "field1", "obj.subfield").includeDefaults(true).get();
+            .setFields("num", "field1", "field2", "obj.subfield").includeDefaults(true).get();
 
         assertThat((Map<String, Object>) response.fieldMappings("test", "type", "num").sourceAsMap().get("num"),
             hasEntry("index", Boolean.TRUE));
@@ -160,6 +162,8 @@ public class SimpleGetFieldMappingsIT extends ESIntegTestCase {
             hasEntry("index", Boolean.TRUE));
         assertThat((Map<String, Object>) response.fieldMappings("test", "type", "field1").sourceAsMap().get("field1"),
             hasEntry("type", "text"));
+        assertThat((Map<String, Object>) response.fieldMappings("test", "type", "field2").sourceAsMap().get("field2"),
+            hasEntry("type", "text"));
         assertThat((Map<String, Object>) response.fieldMappings("test", "type", "obj.subfield").sourceAsMap().get("subfield"),
             hasEntry("type", "keyword"));
     }

server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocValuesFieldExistsQuery;
@@ -61,6 +62,7 @@ public abstract class AbstractGeometryFieldMapper<Parsed, Processed> extends Fie
         static {
             FIELD_TYPE.setStored(false);
             FIELD_TYPE.setOmitNorms(true);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
             FIELD_TYPE.freeze();
         }
     }

server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -22,6 +22,7 @@ import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.document.LatLonPoint;
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Explicit;
@@ -49,6 +50,7 @@ public class GeoPointFieldMapper extends AbstractPointGeometryFieldMapper<List<?
     public static final FieldType FIELD_TYPE = new FieldType();
     static {
         FIELD_TYPE.setStored(false);
+        FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
         FIELD_TYPE.freeze();
     }
 

server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java

@@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.LatLonShape;
+import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.geo.GeometryParser;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -54,6 +55,7 @@ public class GeoShapeFieldMapper extends AbstractShapeGeometryFieldMapper<Geomet
     public static final FieldType FIELD_TYPE = new FieldType();
     static {
         FIELD_TYPE.setDimensions(7, 4, Integer.BYTES);
+        FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
         FIELD_TYPE.setOmitNorms(true);
         FIELD_TYPE.freeze();
     }

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -23,6 +23,7 @@ import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocValuesFieldExistsQuery;
@@ -63,6 +64,7 @@ public class IpFieldMapper extends FieldMapper {
         public static final FieldType FIELD_TYPE = new FieldType();
         static {
             FIELD_TYPE.setDimensions(1, Integer.BYTES);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
             FIELD_TYPE.freeze();
         }
     }

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -428,7 +428,8 @@ public final class KeywordFieldMapper extends FieldMapper {
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);
 
-        if (includeDefaults || (mappedFieldType.isSearchable() && fieldType.indexOptions() != IndexOptions.DOCS)) {
+        if (fieldType.indexOptions() != IndexOptions.NONE
+            && (includeDefaults || fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions())) {
             builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
         }
         if (nullValue != null) {

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -79,6 +79,7 @@ public class NumberFieldMapper extends FieldMapper {
         public static final FieldType FIELD_TYPE = new FieldType();
         static {
             FIELD_TYPE.setStored(false);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
             FIELD_TYPE.freeze();
         }
     }

server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DocValuesFieldExistsQuery;
@@ -75,6 +76,7 @@ public class RangeFieldMapper extends FieldMapper {
         public static final FieldType FIELD_TYPE = new FieldType();
         static {
             FIELD_TYPE.setStored(false);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
             FIELD_TYPE.freeze();
         }
         public static final DateFormatter DATE_FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -920,7 +920,8 @@ public class TextFieldMapper extends FieldMapper {
     @Override
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);
-        if (includeDefaults || (mappedFieldType.isSearchable() && fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions())) {
+        if (fieldType.indexOptions() != IndexOptions.NONE
+            && (includeDefaults || fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions())) {
             builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
         }
         if (includeDefaults || fieldType.storeTermVectors() != Defaults.FIELD_TYPE.storeTermVectors()) {

server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java

@@ -47,7 +47,7 @@ public class BinaryFieldMapperTests extends FieldMapperTestCase<BinaryFieldMappe
 
     @Override
     protected Set<String> unsupportedProperties() {
-        return org.elasticsearch.common.collect.Set.of("analyzer", "eager_global_ordinals", "norms", "similarity");
+        return org.elasticsearch.common.collect.Set.of("analyzer", "eager_global_ordinals", "norms", "similarity", "index");
     }
 
     @Override

server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java

@@ -83,7 +83,7 @@ public class CompletionFieldMapperTests extends FieldMapperTestCase<CompletionFi
 
     @Override
     protected Set<String> unsupportedProperties() {
-        return org.elasticsearch.common.collect.Set.of("doc_values");
+        return org.elasticsearch.common.collect.Set.of("doc_values", "index");
     }
 
     @Override

test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java

@@ -95,6 +95,7 @@ public abstract class FieldMapperTestCase<T extends FieldMapper.Builder<?>> exte
             b.docValues(false);
         }),
         booleanModifier("eager_global_ordinals", true, (a, t) -> a.setEagerGlobalOrdinals(t)),
+        booleanModifier("index", false, (a, t) -> a.index(t)),
         booleanModifier("norms", false, FieldMapper.Builder::omitNorms),
         new Modifier("search_analyzer", true, (a, b) -> {
             a.searchAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()));
@@ -228,22 +229,27 @@ public abstract class FieldMapperTestCase<T extends FieldMapper.Builder<?>> exte
 
         Mapper.BuilderContext context = new Mapper.BuilderContext(SETTINGS, new ContentPath(1));
 
-        XContentBuilder x = JsonXContent.contentBuilder();
+        String mappings = mappingsToString(builder.build(context), false);
-        x.startObject().startObject("properties");
+        String mappingsWithDefault = mappingsToString(builder.build(context), true);
-        builder.build(context).toXContent(x, ToXContent.EMPTY_PARAMS);
-        x.endObject().endObject();
-        String mappings = Strings.toString(x);
 
         mapperService.merge("_doc", new CompressedXContent(mappings), MapperService.MergeReason.MAPPING_UPDATE);
 
         Mapper rebuilt = mapperService.documentMapper().mappers().getMapper(builder.name);
-        x = JsonXContent.contentBuilder();
+        String reparsed = mappingsToString(rebuilt, false);
-        x.startObject().startObject("properties");
+        String reparsedWithDefault = mappingsToString(rebuilt, true);
-        rebuilt.toXContent(x, ToXContent.EMPTY_PARAMS);
-        x.endObject().endObject();
-        String reparsed = Strings.toString(x);
 
         assertThat(reparsed, equalTo(mappings));
+        assertThat(reparsedWithDefault, equalTo(mappingsWithDefault));
+    }
+
+    private String mappingsToString(ToXContent builder, boolean includeDefaults) throws IOException {
+        ToXContent.Params params = includeDefaults ?
+            new ToXContent.MapParams(Collections.singletonMap("include_defaults", "true")) : ToXContent.EMPTY_PARAMS;
+        XContentBuilder x = JsonXContent.contentBuilder();
+        x.startObject().startObject("properties");
+        builder.toXContent(x, params);
+        x.endObject().endObject();
+        return Strings.toString(x);
     }
 
 }

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java

@@ -37,7 +37,7 @@ public class HistogramFieldMapperTests extends FieldMapperTestCase<HistogramFiel
 
     @Override
     protected Set<String> unsupportedProperties() {
-        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store");
+        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store", "index");
     }
 
     public void testParseValue() throws Exception {

x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java

@@ -69,6 +69,8 @@ public class ConstantKeywordFieldMapper extends FieldMapper {
             builder = this;
         }
 
+        // TODO we should ban setting 'index' on constant keyword
+
         public Builder setValue(String value) {
             this.value = value;
             return this;

x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java

@@ -36,7 +36,7 @@ public class ConstantKeywordFieldMapperTests extends FieldMapperTestCase<Constan
 
     @Override
     protected Set<String> unsupportedProperties() {
-        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values");
+        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values", "index");
     }
 
     @Override

x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java

@@ -633,7 +633,8 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper {
     @Override
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);
-        if (includeDefaults || mappedFieldType.isSearchable() && fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions()) {
+        if (fieldType.indexOptions() != IndexOptions.NONE
+            && (includeDefaults || fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions())) {
             builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
         }
         if (includeDefaults || depthLimit != Defaults.DEPTH_LIMIT) {

x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java

@@ -9,6 +9,7 @@ package org.elasticsearch.xpack.spatial.index.mapper;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.LatLonShape;
 import org.apache.lucene.document.ShapeField;
+import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -60,13 +61,17 @@ import java.util.Map;
  */
 public class GeoShapeWithDocValuesFieldMapper extends GeoShapeFieldMapper {
     public static final String CONTENT_TYPE = "geo_shape";
+    public static final FieldType FIELD_TYPE = new FieldType();
+    static {
+        FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+    }
 
     public static class Builder extends AbstractShapeGeometryFieldMapper.Builder<Builder, GeoShapeWithDocValuesFieldType> {
 
         private boolean docValuesSet = false;
 
         public Builder(String name) {
-            super (name, new FieldType());
+            super (name, FIELD_TYPE);
             this.hasDocValues = true;
         }
 

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java

@@ -52,7 +52,7 @@ public class DenseVectorFieldMapperTests extends FieldMapperTestCase<DenseVector
 
     @Override
     protected Set<String> unsupportedProperties() {
-        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store");
+        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store", "index");
     }
 
     @Before

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java

@@ -47,7 +47,7 @@ public class SparseVectorFieldMapperTests extends FieldMapperTestCase<SparseVect
 
     @Override
     protected Set<String> unsupportedProperties() {
-        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store");
+        return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store", "index");
     }
 
     @Before