honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add serialization test for FieldMappers when include_defaults=true (#58235) (#58328) 

Fixes a bug in TextFieldMapper serialization when index is false, and adds a
base-class test to ensure that all field mappers are tested against all variations
with defaults both included and excluded.

Fixes #58188
This commit is contained in:
Alan Woodward 2020-06-18 15:46:04 +01:00 committed by GitHub
parent 50b391e91b
commit 4b8cf2af6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 61 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
View File

@ -72,6 +72,10 @@ public class ScaledFloatFieldMapper extends FieldMapper {
public static final String CONTENT_TYPE = "scaled_float";
// use the same default as numbers
private static final Setting<Boolean> COERCE_SETTING = NumberFieldMapper.COERCE_SETTING;
private static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
}
public static class Builder extends FieldMapper.Builder<Builder> {
@ -82,7 +86,7 @@ public class ScaledFloatFieldMapper extends FieldMapper {
private Double nullValue;
public Builder(String name) {
super(name, new FieldType());
super(name, FIELD_TYPE);
builder = this;
}

2
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java
View File

@ -45,7 +45,7 @@ public class RankFeatureFieldMapperTests extends FieldMapperTestCase<RankFeature
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values");
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values", "index");
}
@Before

2
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java
View File

@ -40,7 +40,7 @@ public class RankFeaturesFieldMapperTests extends FieldMapperTestCase<RankFeatur
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values");
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values", "index");
}
IndexService indexService;

2
plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java
View File

@ -658,7 +658,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);
if (includeDefaults || (mappedFieldType.isSearchable() && fieldType.indexOptions() != IndexOptions.DOCS)) {
if (fieldType.indexOptions() != IndexOptions.NONE && (includeDefaults || fieldType.indexOptions() != IndexOptions.DOCS)) {
builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
}
if (nullValue != null) {

2
plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
View File

@ -58,7 +58,7 @@ public class Murmur3FieldMapperTests extends FieldMapperTestCase<Murmur3FieldMap
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values");
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "index");
}
@Before

6
server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java
View File

@ -148,9 +148,11 @@ public class SimpleGetFieldMappingsIT extends ESIntegTestCase {
public void testSimpleGetFieldMappingsWithDefaults() throws Exception {
assertAcked(prepareCreate("test").addMapping("type", getMappingForType("type")));
client().admin().indices().preparePutMapping("test").setType("type").setSource("num", "type=long").get();
client().admin().indices().preparePutMapping("test").setType("type")
.setSource("field2", "type=text,index=false").get();
GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings()
.setFields("num", "field1", "obj.subfield").includeDefaults(true).get();
.setFields("num", "field1", "field2", "obj.subfield").includeDefaults(true).get();
assertThat((Map<String, Object>) response.fieldMappings("test", "type", "num").sourceAsMap().get("num"),
hasEntry("index", Boolean.TRUE));
@ -160,6 +162,8 @@ public class SimpleGetFieldMappingsIT extends ESIntegTestCase {
hasEntry("index", Boolean.TRUE));
assertThat((Map<String, Object>) response.fieldMappings("test", "type", "field1").sourceAsMap().get("field1"),
hasEntry("type", "text"));
assertThat((Map<String, Object>) response.fieldMappings("test", "type", "field2").sourceAsMap().get("field2"),
hasEntry("type", "text"));
assertThat((Map<String, Object>) response.fieldMappings("test", "type", "obj.subfield").sourceAsMap().get("subfield"),
hasEntry("type", "keyword"));
}

2
server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
@ -61,6 +62,7 @@ public abstract class AbstractGeometryFieldMapper<Parsed, Processed> extends Fie
static {
FIELD_TYPE.setStored(false);
FIELD_TYPE.setOmitNorms(true);
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
FIELD_TYPE.freeze();
}
}

2
server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
View File

@ -22,6 +22,7 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.LatLonDocValuesField;
import org.apache.lucene.document.LatLonPoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Explicit;
@ -49,6 +50,7 @@ public class GeoPointFieldMapper extends AbstractPointGeometryFieldMapper<List<?
public static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setStored(false);
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
FIELD_TYPE.freeze();
}

2
server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.LatLonShape;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.geo.GeometryParser;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
@ -54,6 +55,7 @@ public class GeoShapeFieldMapper extends AbstractShapeGeometryFieldMapper<Geomet
public static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setDimensions(7, 4, Integer.BYTES);
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
FIELD_TYPE.setOmitNorms(true);
FIELD_TYPE.freeze();
}

2
server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
@ -63,6 +64,7 @@ public class IpFieldMapper extends FieldMapper {
public static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setDimensions(1, Integer.BYTES);
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
FIELD_TYPE.freeze();
}
}

3
server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
View File

@ -428,7 +428,8 @@ public final class KeywordFieldMapper extends FieldMapper {
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);
if (includeDefaults || (mappedFieldType.isSearchable() && fieldType.indexOptions() != IndexOptions.DOCS)) {
if (fieldType.indexOptions() != IndexOptions.NONE
&& (includeDefaults || fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions())) {
builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
}
if (nullValue != null) {

1
server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
View File

@ -79,6 +79,7 @@ public class NumberFieldMapper extends FieldMapper {
public static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setStored(false);
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
FIELD_TYPE.freeze();
}
}

2
server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
@ -75,6 +76,7 @@ public class RangeFieldMapper extends FieldMapper {
public static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setStored(false);
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
FIELD_TYPE.freeze();
}
public static final DateFormatter DATE_FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;

3
server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
View File

@ -920,7 +920,8 @@ public class TextFieldMapper extends FieldMapper {
@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);
if (includeDefaults || (mappedFieldType.isSearchable() && fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions())) {
if (fieldType.indexOptions() != IndexOptions.NONE
&& (includeDefaults || fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions())) {
builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
}
if (includeDefaults || fieldType.storeTermVectors() != Defaults.FIELD_TYPE.storeTermVectors()) {

2
server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
View File

@ -47,7 +47,7 @@ public class BinaryFieldMapperTests extends FieldMapperTestCase<BinaryFieldMappe
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("analyzer", "eager_global_ordinals", "norms", "similarity");
return org.elasticsearch.common.collect.Set.of("analyzer", "eager_global_ordinals", "norms", "similarity", "index");
}
@Override

2
server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
View File

@ -83,7 +83,7 @@ public class CompletionFieldMapperTests extends FieldMapperTestCase<CompletionFi
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("doc_values");
return org.elasticsearch.common.collect.Set.of("doc_values", "index");
}
@Override

26
test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java
View File

@ -95,6 +95,7 @@ public abstract class FieldMapperTestCase<T extends FieldMapper.Builder<?>> exte
b.docValues(false);
}),
booleanModifier("eager_global_ordinals", true, (a, t) -> a.setEagerGlobalOrdinals(t)),
booleanModifier("index", false, (a, t) -> a.index(t)),
booleanModifier("norms", false, FieldMapper.Builder::omitNorms),
new Modifier("search_analyzer", true, (a, b) -> {
a.searchAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()));
@ -228,22 +229,27 @@ public abstract class FieldMapperTestCase<T extends FieldMapper.Builder<?>> exte
Mapper.BuilderContext context = new Mapper.BuilderContext(SETTINGS, new ContentPath(1));
XContentBuilder x = JsonXContent.contentBuilder();
x.startObject().startObject("properties");
builder.build(context).toXContent(x, ToXContent.EMPTY_PARAMS);
x.endObject().endObject();
String mappings = Strings.toString(x);
String mappings = mappingsToString(builder.build(context), false);
String mappingsWithDefault = mappingsToString(builder.build(context), true);
mapperService.merge("_doc", new CompressedXContent(mappings), MapperService.MergeReason.MAPPING_UPDATE);
Mapper rebuilt = mapperService.documentMapper().mappers().getMapper(builder.name);
x = JsonXContent.contentBuilder();
x.startObject().startObject("properties");
rebuilt.toXContent(x, ToXContent.EMPTY_PARAMS);
x.endObject().endObject();
String reparsed = Strings.toString(x);
String reparsed = mappingsToString(rebuilt, false);
String reparsedWithDefault = mappingsToString(rebuilt, true);
assertThat(reparsed, equalTo(mappings));
assertThat(reparsedWithDefault, equalTo(mappingsWithDefault));
}
private String mappingsToString(ToXContent builder, boolean includeDefaults) throws IOException {
ToXContent.Params params = includeDefaults ?
new ToXContent.MapParams(Collections.singletonMap("include_defaults", "true")) : ToXContent.EMPTY_PARAMS;
XContentBuilder x = JsonXContent.contentBuilder();
x.startObject().startObject("properties");
builder.toXContent(x, params);
x.endObject().endObject();
return Strings.toString(x);
}
}

2
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java
View File

@ -37,7 +37,7 @@ public class HistogramFieldMapperTests extends FieldMapperTestCase<HistogramFiel
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store");
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store", "index");
}
public void testParseValue() throws Exception {

2
x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java
View File

@ -69,6 +69,8 @@ public class ConstantKeywordFieldMapper extends FieldMapper {
builder = this;
}
// TODO we should ban setting 'index' on constant keyword
public Builder setValue(String value) {
this.value = value;
return this;

2
x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java
View File

@ -36,7 +36,7 @@ public class ConstantKeywordFieldMapperTests extends FieldMapperTestCase<Constan
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values");
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "store", "doc_values", "index");
}
@Override

3
x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java
View File

@ -633,7 +633,8 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper {
@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);
if (includeDefaults || mappedFieldType.isSearchable() && fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions()) {
if (fieldType.indexOptions() != IndexOptions.NONE
&& (includeDefaults || fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions())) {
builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
}
if (includeDefaults || depthLimit != Defaults.DEPTH_LIMIT) {

7
x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java
View File

@ -9,6 +9,7 @@ package org.elasticsearch.xpack.spatial.index.mapper;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.LatLonShape;
import org.apache.lucene.document.ShapeField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
@ -60,13 +61,17 @@ import java.util.Map;
*/
public class GeoShapeWithDocValuesFieldMapper extends GeoShapeFieldMapper {
public static final String CONTENT_TYPE = "geo_shape";
public static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
}
public static class Builder extends AbstractShapeGeometryFieldMapper.Builder<Builder, GeoShapeWithDocValuesFieldType> {
private boolean docValuesSet = false;
public Builder(String name) {
super (name, new FieldType());
super (name, FIELD_TYPE);
this.hasDocValues = true;
}

2
x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java
View File

@ -52,7 +52,7 @@ public class DenseVectorFieldMapperTests extends FieldMapperTestCase<DenseVector
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store");
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store", "index");
}
@Before

2
x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java
View File

@ -47,7 +47,7 @@ public class SparseVectorFieldMapperTests extends FieldMapperTestCase<SparseVect
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store");
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store", "index");
}
@Before