honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make StringFieldMapper.toXContent aware of defaults for not_analyzed fields. 

StringFieldMapper.toXContent uses the defaults for analyzed fields in order to
know which options to add to the builder. This means that if the field is not
analyzed and has norms enabled, it will omit to emit `norms.enabled: true`.
Parsing the mapping again will result in a StringFieldMapper that has norms
disabled.

The same fix applies to index options.

Close #4760
This commit is contained in:
Adrien Grand 2014-01-20 13:14:03 +01:00
parent 6469f6ef06
commit 3586157467
2 changed files with 65 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
View File

@ -126,7 +126,12 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
// if the field is not analyzed, then by default, we should omit norms and have docs only // if the field is not analyzed, then by default, we should omit norms and have docs only
// index options, as probably what the user really wants // index options, as probably what the user really wants
// if they are set explicitly, we will use those values // if they are set explicitly, we will use those values
// we also change the values on the default field type so that toXContent emits what
// differs from the defaults
FieldType defaultFieldType = new FieldType(Defaults.FIELD_TYPE);
if (fieldType.indexed() && !fieldType.tokenized()) { if (fieldType.indexed() && !fieldType.tokenized()) {
defaultFieldType.setOmitNorms(true);
defaultFieldType.setIndexOptions(IndexOptions.DOCS_ONLY);
if (!omitNormsSet && boost == Defaults.BOOST) { if (!omitNormsSet && boost == Defaults.BOOST) {
fieldType.setOmitNorms(true); fieldType.setOmitNorms(true);
} }
@ -134,8 +139,9 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
fieldType.setIndexOptions(IndexOptions.DOCS_ONLY); fieldType.setIndexOptions(IndexOptions.DOCS_ONLY);
} }
} }
defaultFieldType.freeze();
StringFieldMapper fieldMapper = new StringFieldMapper(buildNames(context), StringFieldMapper fieldMapper = new StringFieldMapper(buildNames(context),
boost, fieldType, docValues, nullValue, indexAnalyzer, searchAnalyzer, searchQuotedAnalyzer, boost, fieldType, defaultFieldType, docValues, nullValue, indexAnalyzer, searchAnalyzer, searchQuotedAnalyzer,
positionOffsetGap, ignoreAbove, postingsProvider, docValuesProvider, similarity, normsLoading, positionOffsetGap, ignoreAbove, postingsProvider, docValuesProvider, similarity, normsLoading,
fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context));
fieldMapper.includeInAll(includeInAll); fieldMapper.includeInAll(includeInAll);
@ -183,16 +189,13 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
} }
private String nullValue; private String nullValue;
private Boolean includeInAll; private Boolean includeInAll;
private int positionOffsetGap; private int positionOffsetGap;
private NamedAnalyzer searchQuotedAnalyzer; private NamedAnalyzer searchQuotedAnalyzer;
private int ignoreAbove; private int ignoreAbove;
private final FieldType defaultFieldType;
protected StringFieldMapper(Names names, float boost, FieldType fieldType, Boolean docValues, protected StringFieldMapper(Names names, float boost, FieldType fieldType,FieldType defaultFieldType, Boolean docValues,
String nullValue, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, String nullValue, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
NamedAnalyzer searchQuotedAnalyzer, int positionOffsetGap, int ignoreAbove, NamedAnalyzer searchQuotedAnalyzer, int positionOffsetGap, int ignoreAbove,
PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat, PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat,
@ -203,6 +206,7 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
if (fieldType.tokenized() && fieldType.indexed() && hasDocValues()) { if (fieldType.tokenized() && fieldType.indexed() && hasDocValues()) {
throw new MapperParsingException("Field [" + names.fullName() + "] cannot be analyzed and have doc values"); throw new MapperParsingException("Field [" + names.fullName() + "] cannot be analyzed and have doc values");
} }
this.defaultFieldType = defaultFieldType;
this.nullValue = nullValue; this.nullValue = nullValue;
this.positionOffsetGap = positionOffsetGap; this.positionOffsetGap = positionOffsetGap;
this.searchQuotedAnalyzer = searchQuotedAnalyzer != null ? searchQuotedAnalyzer : this.searchAnalyzer; this.searchQuotedAnalyzer = searchQuotedAnalyzer != null ? searchQuotedAnalyzer : this.searchAnalyzer;
@ -211,7 +215,7 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
@Override @Override
public FieldType defaultFieldType() { public FieldType defaultFieldType() {
return Defaults.FIELD_TYPE; return defaultFieldType;
} }
@Override @Override

73
src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java
View File

@ -19,11 +19,14 @@
package org.elasticsearch.index.mapper.string; package org.elasticsearch.index.mapper.string;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.DocValuesType; import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.*;
@ -74,6 +77,35 @@ public class SimpleStringMappingTests extends ElasticsearchTestCase {
assertThat(doc.rootDoc().getField("field"), nullValue()); assertThat(doc.rootDoc().getField("field"), nullValue());
} }
private void assertDefaultAnalyzedFieldType(IndexableFieldType fieldType) {
assertThat(fieldType.omitNorms(), equalTo(false));
assertThat(fieldType.indexOptions(), equalTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS));
assertThat(fieldType.storeTermVectors(), equalTo(false));
assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
}
private void assertEquals(IndexableFieldType ft1, IndexableFieldType ft2) {
assertEquals(ft1.indexed(), ft2.indexed());
assertEquals(ft1.tokenized(), ft2.tokenized());
assertEquals(ft1.omitNorms(), ft2.omitNorms());
assertEquals(ft1.indexOptions(), ft2.indexOptions());
assertEquals(ft1.storeTermVectors(), ft2.storeTermVectors());
assertEquals(ft1.docValueType(), ft2.docValueType());
}
private void assertParseIdemPotent(IndexableFieldType expected, DocumentMapper mapper) throws Exception {
String mapping = mapper.toXContent(XContentFactory.jsonBuilder().startObject(), new ToXContent.MapParams(ImmutableMap.<String, String>of())).endObject().string();
mapper = MapperTestUtils.newParser().parse(mapping);
ParsedDocument doc = mapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field", "2345")
.endObject()
.bytes());
assertEquals(expected, doc.rootDoc().getField("field").fieldType());
}
@Test @Test
public void testDefaultsForAnalyzed() throws Exception { public void testDefaultsForAnalyzed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
@ -88,12 +120,9 @@ public class SimpleStringMappingTests extends ElasticsearchTestCase {
.endObject() .endObject()
.bytes()); .bytes());
assertThat(doc.rootDoc().getField("field").fieldType().omitNorms(), equalTo(false)); IndexableFieldType fieldType = doc.rootDoc().getField("field").fieldType();
assertThat(doc.rootDoc().getField("field").fieldType().indexOptions(), equalTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)); assertDefaultAnalyzedFieldType(fieldType);
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectors(), equalTo(false)); assertParseIdemPotent(fieldType, defaultMapper);
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorOffsets(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPositions(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPayloads(), equalTo(false));
} }
@Test @Test
@ -110,12 +139,14 @@ public class SimpleStringMappingTests extends ElasticsearchTestCase {
.endObject() .endObject()
.bytes()); .bytes());
assertThat(doc.rootDoc().getField("field").fieldType().omitNorms(), equalTo(true)); IndexableFieldType fieldType = doc.rootDoc().getField("field").fieldType();
assertThat(doc.rootDoc().getField("field").fieldType().indexOptions(), equalTo(FieldInfo.IndexOptions.DOCS_ONLY)); assertThat(fieldType.omitNorms(), equalTo(true));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectors(), equalTo(false)); assertThat(fieldType.indexOptions(), equalTo(FieldInfo.IndexOptions.DOCS_ONLY));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorOffsets(), equalTo(false)); assertThat(fieldType.storeTermVectors(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPositions(), equalTo(false)); assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPayloads(), equalTo(false)); assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
assertParseIdemPotent(fieldType, defaultMapper);
// now test it explicitly set // now test it explicitly set
@ -131,12 +162,14 @@ public class SimpleStringMappingTests extends ElasticsearchTestCase {
.endObject() .endObject()
.bytes()); .bytes());
assertThat(doc.rootDoc().getField("field").fieldType().omitNorms(), equalTo(false)); fieldType = doc.rootDoc().getField("field").fieldType();
assertThat(doc.rootDoc().getField("field").fieldType().indexOptions(), equalTo(FieldInfo.IndexOptions.DOCS_AND_FREQS)); assertThat(fieldType.omitNorms(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectors(), equalTo(false)); assertThat(fieldType.indexOptions(), equalTo(FieldInfo.IndexOptions.DOCS_AND_FREQS));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorOffsets(), equalTo(false)); assertThat(fieldType.storeTermVectors(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPositions(), equalTo(false)); assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPayloads(), equalTo(false)); assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
assertParseIdemPotent(fieldType, defaultMapper);
// also test the deprecated omit_norms // also test the deprecated omit_norms
@ -152,7 +185,9 @@ public class SimpleStringMappingTests extends ElasticsearchTestCase {
.endObject() .endObject()
.bytes()); .bytes());
assertThat(doc.rootDoc().getField("field").fieldType().omitNorms(), equalTo(false)); fieldType = doc.rootDoc().getField("field").fieldType();
assertThat(fieldType.omitNorms(), equalTo(false));
assertParseIdemPotent(fieldType, defaultMapper);
} }
@Test @Test