honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cut over more mapping tests to MapperServiceTestCase (#62312) 

Shaves a few more seconds off the build.
This commit is contained in:
Alan Woodward 2020-09-14 15:32:35 +01:00 committed by Alan Woodward
parent f4dfdc9d59
commit 5358cee29c
7 changed files with 452 additions and 848 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java
View File

@ -42,6 +42,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicReference;
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING;
import static org.hamcrest.Matchers.equalTo;
public class DynamicMappingIT extends ESIntegTestCase {
@ -162,4 +163,12 @@ public class DynamicMappingIT extends ESIntegTestCase {
indexingCompletedLatch.countDown();
}
}
public void testMappingVersionAfterDynamicMappingUpdate() {
createIndex("test");
final ClusterService clusterService = internalCluster().clusterService();
final long previousVersion = clusterService.state().metadata().index("test").getMappingVersion();
client().prepareIndex("test", "_doc").setId("1").setSource("field", "text").get();
assertThat(clusterService.state().metadata().index("test").getMappingVersion(), equalTo(1 + previousVersion));
}
}

778
server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
View File

@ -18,86 +18,55 @@
*/
package org.elasticsearch.index.mapper;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
import java.io.IOException;
import java.time.Instant;
import java.util.Collection;
import java.util.Collections;
import static java.util.Collections.emptyMap;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;
public class DynamicMappingTests extends ESSingleNodeTestCase {
public class DynamicMappingTests extends MapperServiceTestCase {
@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return Collections.singleton(InternalSettingsPlugin.class);
private XContentBuilder dynamicMapping(String dynamicValue, CheckedConsumer<XContentBuilder, IOException> buildFields)
throws IOException {
return topMapping(b -> {
b.field("dynamic", dynamicValue);
b.startObject("properties");
buildFields.accept(b);
b.endObject();
});
}
public void testDynamicTrue() throws IOException {
String mapping = Strings.toString(jsonBuilder().startObject().startObject("type")
.field("dynamic", "true")
.startObject("properties")
.startObject("field1").field("type", "text").endObject()
.endObject()
.endObject().endObject());
DocumentMapper defaultMapper = createDocumentMapper(dynamicMapping("true",
b -> b.startObject("field1").field("type", "text").endObject()));
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
.parse("type", new CompressedXContent(mapping));
ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(jsonBuilder()
.startObject()
.field("field1", "value1")
.field("field2", "value2")
.endObject()),
XContentType.JSON));
ParsedDocument doc = defaultMapper.parse(source(b -> {
b.field("field1", "value1");
b.field("field2", "value2");
}));
assertThat(doc.rootDoc().get("field1"), equalTo("value1"));
assertThat(doc.rootDoc().get("field2"), equalTo("value2"));
}
public void testDynamicFalse() throws IOException {
String mapping = Strings.toString(jsonBuilder().startObject().startObject("type")
.field("dynamic", "false")
.startObject("properties")
.startObject("field1").field("type", "text").endObject()
.endObject()
.endObject().endObject());
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
.parse("type", new CompressedXContent(mapping));
DocumentMapper defaultMapper = createDocumentMapper(dynamicMapping("false",
b -> b.startObject("field1").field("type", "text").endObject()));
ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(jsonBuilder()
.startObject()
.field("field1", "value1")
.field("field2", "value2")
.endObject()),
XContentType.JSON));
ParsedDocument doc = defaultMapper.parse(source(b -> {
b.field("field1", "value1");
b.field("field2", "value2");
}));
assertThat(doc.rootDoc().get("field1"), equalTo("value1"));
assertThat(doc.rootDoc().get("field2"), nullValue());
@ -105,416 +74,263 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
public void testDynamicStrict() throws IOException {
String mapping = Strings.toString(jsonBuilder().startObject().startObject("type")
.field("dynamic", "strict")
.startObject("properties")
.startObject("field1").field("type", "text").endObject()
.endObject()
.endObject().endObject());
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
.parse("type", new CompressedXContent(mapping));
DocumentMapper defaultMapper = createDocumentMapper(dynamicMapping("strict",
b -> b.startObject("field1").field("type", "text").endObject()));
StrictDynamicMappingException e = expectThrows(StrictDynamicMappingException.class,
() -> defaultMapper.parse(new SourceToParse("test", "type", "1",
BytesReference.bytes(jsonBuilder()
.startObject()
.field("field1", "value1")
.field("field2", "value2")
.endObject()),
XContentType.JSON)));
assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [type] is not allowed"));
() -> defaultMapper.parse(source(b -> {
b.field("field1", "value1");
b.field("field2", "value2");
})));
assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [_doc] is not allowed"));
e = expectThrows(StrictDynamicMappingException.class,
() -> defaultMapper.parse(new SourceToParse("test", "type", "1",
BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field1", "value1")
.field("field2", (String) null)
.endObject()),
XContentType.JSON)));
assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [type] is not allowed"));
() -> defaultMapper.parse(source(b -> {
b.field("field1", "value1");
b.nullField("field2");
})));
assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [_doc] is not allowed"));
}
public void testDynamicFalseWithInnerObjectButDynamicSetOnRoot() throws IOException {
String mapping = Strings.toString(jsonBuilder().startObject().startObject("type")
.field("dynamic", "false")
.startObject("properties")
.startObject("obj1").startObject("properties")
.startObject("field1").field("type", "text").endObject()
.endObject().endObject()
.endObject()
.endObject().endObject());
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
.parse("type", new CompressedXContent(mapping));
DocumentMapper defaultMapper = createDocumentMapper(dynamicMapping("false", b -> {
b.startObject("obj1");
{
b.startObject("properties");
{
b.startObject("field1").field("type", "text").endObject();
}
b.endObject();
}
b.endObject();
}));
ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1",
BytesReference.bytes(jsonBuilder()
.startObject().startObject("obj1")
.field("field1", "value1")
.field("field2", "value2")
.endObject()
.endObject()),
XContentType.JSON));
ParsedDocument doc = defaultMapper.parse(source(b -> {
b.startObject("obj1");
{
b.field("field1", "value1");
b.field("field2", "value2");
}
b.endObject();
}));
assertThat(doc.rootDoc().get("obj1.field1"), equalTo("value1"));
assertThat(doc.rootDoc().get("obj1.field2"), nullValue());
}
public void testDynamicStrictWithInnerObjectButDynamicSetOnRoot() throws IOException {
String mapping = Strings.toString(jsonBuilder().startObject().startObject("type")
.field("dynamic", "strict")
.startObject("properties")
.startObject("obj1").startObject("properties")
.startObject("field1").field("type", "text").endObject()
.endObject().endObject()
.endObject()
.endObject().endObject());
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
.parse("type", new CompressedXContent(mapping));
DocumentMapper defaultMapper = createDocumentMapper(dynamicMapping("strict", b -> {
b.startObject("obj1");
{
b.startObject("properties");
{
b.startObject("field1").field("type", "text").endObject();
}
b.endObject();
}
b.endObject();
}));
StrictDynamicMappingException e = expectThrows(StrictDynamicMappingException.class, () ->
defaultMapper.parse(new SourceToParse("test", "type", "1",
BytesReference.bytes(jsonBuilder()
.startObject().startObject("obj1")
.field("field1", "value1")
.field("field2", "value2")
.endObject()
.endObject()),
XContentType.JSON)));
defaultMapper.parse(source(b -> {
b.startObject("obj1");
{
b.field("field1", "value1");
b.field("field2", "value2");
}
b.endObject();
})));
assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [obj1] is not allowed"));
}
public void testDynamicMappingOnEmptyString() throws Exception {
IndexService service = createIndex("test");
client().prepareIndex("test", "type").setSource("empty_field", "").get();
MappedFieldType fieldType = service.mapperService().fieldType("empty_field");
MapperService mapperService = createMapperService(mapping(b -> {}));
ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("empty_field", "")));
merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate()));
MappedFieldType fieldType = mapperService.fieldType("empty_field");
assertNotNull(fieldType);
}
private String serialize(ToXContent mapper) throws Exception {
XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
mapper.toXContent(builder, new ToXContent.MapParams(emptyMap()));
return Strings.toString(builder.endObject());
}
private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception {
IndexMetadata build = IndexMetadata.builder("")
.settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
.numberOfShards(1).numberOfReplicas(0).build();
IndexSettings settings = new IndexSettings(build, Settings.EMPTY);
SourceToParse source = new SourceToParse("test", mapper.type(), "some_id",
BytesReference.bytes(builder), builder.contentType());
try (XContentParser xContentParser = createParser(JsonXContent.jsonXContent, source.source())) {
ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext(settings, parser, mapper, source, xContentParser);
assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
ctx.parser().nextToken();
DocumentParser.parseObjectOrNested(ctx, mapper.root());
Mapping mapping = DocumentParser.createDynamicUpdate(mapper.mapping(), mapper, ctx.getDynamicMappers());
return mapping == null ? null : mapping.root();
}
}
public void testDynamicMappingsNotNeeded() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("foo").field("type", "text").endObject().endObject()
.endObject().endObject());
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject());
// foo is already defined in the mappings
assertNull(update);
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "text")));
ParsedDocument doc = mapper.parse(source(b -> b.field("field", "bar")));
// field is already defined in mappings
assertNull(doc.dynamicMappingsUpdate());
}
public void testField() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type").endObject()
.endObject());
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, serialize(mapper));
DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo")
.field("type", "text")
.startObject("fields")
.startObject("keyword")
.field("type", "keyword")
.field("ignore_above", 256)
.endObject()
.endObject()
.endObject()
.endObject().endObject().endObject()), serialize(update));
ParsedDocument doc = mapper.parse(source(b -> b.field("foo", "bar")));
assertNotNull(doc.dynamicMappingsUpdate());
assertEquals(
"{\"_doc\":{\"properties\":{\"foo\":{\"type\":\"text\",\"fields\":" +
"{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}}",
Strings.toString(doc.dynamicMappingsUpdate()));
}
public void testIncremental() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
// Make sure that mapping updates are incremental, this is important for performance otherwise
// every new field introduction runs in linear time with the total number of fields
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("foo").field("type", "text").endObject().endObject()
.endObject().endObject());
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "text")));
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, serialize(mapper));
ParsedDocument doc = mapper.parse(source(b -> {
b.field("field", "bar");
b.field("bar", "baz");
}));
assertNotNull(doc.dynamicMappingsUpdate());
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar")
.field("bar", "baz").endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
// foo is NOT in the update
.startObject("bar").field("type", "text")
.startObject("fields")
.startObject("keyword")
.field("type", "keyword")
.field("ignore_above", 256)
.endObject()
.endObject()
.endObject()
.endObject().endObject().endObject()), serialize(update));
assertThat(Strings.toString(doc.dynamicMappingsUpdate()), containsString("{\"bar\":"));
// field is NOT in the update
assertThat(Strings.toString(doc.dynamicMappingsUpdate()), not(containsString("{\"field\":")));
}
public void testIntroduceTwoFields() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type").endObject()
.endObject());
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, serialize(mapper));
DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
ParsedDocument doc = mapper.parse(source(b -> {
b.field("foo", "bar");
b.field("bar", "baz");
}));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar")
.field("bar", "baz").endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("bar").field("type", "text")
.startObject("fields")
.startObject("keyword")
.field("type", "keyword")
.field("ignore_above", 256)
.endObject()
.endObject()
.endObject()
.startObject("foo").field("type", "text")
.startObject("fields")
.startObject("keyword")
.field("type", "keyword")
.field("ignore_above", 256)
.endObject()
.endObject()
.endObject()
.endObject().endObject().endObject()), serialize(update));
assertNotNull(doc.dynamicMappingsUpdate());
assertThat(Strings.toString(doc.dynamicMappingsUpdate()), containsString("\"foo\":{"));
assertThat(Strings.toString(doc.dynamicMappingsUpdate()), containsString("\"bar\":{"));
}
public void testObject() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type").endObject()
.endObject());
DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
ParsedDocument doc = mapper.parse(source(b -> {
b.startObject("foo");
{
b.startObject("bar").field("baz", "foo").endObject();
}
b.endObject();
}));
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, serialize(mapper));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo").startObject("bar")
.field("baz", "foo").endObject().endObject().endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
String serializedUpdate = serialize(update);
assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz")
.field("type", "text")
.startObject("fields").startObject("keyword").field("type", "keyword")
.field("ignore_above", 256).endObject()
.endObject().endObject().endObject().endObject().endObject().endObject()
.endObject().endObject().endObject()), serialize(update));
assertNotNull(doc.dynamicMappingsUpdate());
assertThat(Strings.toString(doc.dynamicMappingsUpdate()),
containsString("{\"foo\":{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"text\""));
}
public void testArray() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type").endObject()
.endObject());
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, serialize(mapper));
DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo").value("bar").value("baz").endArray()));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject()
.startArray("foo").value("bar").value("baz").endArray().endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo")
.field("type", "text")
.startObject("fields")
.startObject("keyword")
.field("type", "keyword")
.field("ignore_above", 256)
.endObject()
.endObject()
.endObject()
.endObject().endObject().endObject()), serialize(update));
assertNotNull(doc.dynamicMappingsUpdate());
assertThat(Strings.toString(doc.dynamicMappingsUpdate()),
containsString("{\"foo\":{\"type\":\"text\""));
}
public void testInnerDynamicMapping() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties")
.startObject("foo").field("type", "object").endObject()
.endObject().endObject().endObject());
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, serialize(mapper));
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "object")));
ParsedDocument doc = mapper.parse(source(b -> {
b.startObject("field");
{
b.startObject("bar").field("baz", "foo").endObject();
}
b.endObject();
}));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo")
.startObject("bar").field("baz", "foo").endObject().endObject().endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo").startObject("properties").startObject("bar").startObject("properties")
.startObject("baz").field("type", "text").startObject("fields")
.startObject("keyword").field("type", "keyword").field("ignore_above", 256).endObject()
.endObject().endObject().endObject().endObject().endObject().endObject()
.endObject().endObject().endObject()), serialize(update));
assertNotNull(doc.dynamicMappingsUpdate());
assertThat(Strings.toString(doc.dynamicMappingsUpdate()),
containsString("{\"field\":{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"text\""));
}
public void testComplexArray() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type").endObject()
.endObject());
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, serialize(mapper));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startArray("foo")
.startObject().field("bar", "baz").endObject()
.startObject().field("baz", 3).endObject()
.endArray().endObject());
assertEquals(mapping, serialize(mapper));
assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo").startObject("properties")
.startObject("bar").field("type", "text")
.startObject("fields")
.startObject("keyword")
.field("type", "keyword")
.field("ignore_above", 256)
.endObject()
.endObject()
.endObject()
.startObject("baz").field("type", "long").endObject()
.endObject().endObject()
.endObject().endObject().endObject()), serialize(update));
DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
ParsedDocument doc = mapper.parse(source(b -> {
b.startArray("foo");
{
b.startObject().field("bar", "baz").endObject();
b.startObject().field("baz", 3).endObject();
}
b.endArray();
}));
assertNotNull(doc.dynamicMappingsUpdate());
assertEquals("{\"_doc\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\":\"text\",\"fields\":{" +
"\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"baz\":{\"type\":\"long\"}}}}}}",
Strings.toString(doc.dynamicMappingsUpdate()));
}
public void testReuseExistingMappings() throws Exception {
IndexService indexService = createIndex("test", Settings.EMPTY, "type",
"my_field1", "type=text,store=true",
"my_field2", "type=integer,store=false",
"my_field3", "type=long,doc_values=false",
"my_field4", "type=float,index=false",
"my_field5", "type=double,store=true",
"my_field6", "type=date,doc_values=false",
"my_field7", "type=boolean,doc_values=false");
// Even if the dynamic type of our new field is long, we already have a mapping for the same field
// of type string so it should be mapped as a string
DocumentMapper newMapper = indexService.mapperService().documentMapperWithAutoCreate("type").getDocumentMapper();
Mapper update = parse(newMapper, indexService.mapperService().documentMapperParser(),
XContentFactory.jsonBuilder().startObject()
.field("my_field1", 42)
.field("my_field2", 43)
.field("my_field3", 44)
.field("my_field4", 45)
.field("my_field5", 46)
.field("my_field6", Instant.now().toEpochMilli())
.field("my_field7", true)
.endObject());
assertNull(update);
DocumentMapper newMapper = createDocumentMapper(mapping(b -> {
b.startObject("my_field1").field("type", "text").field("store", "true").endObject();
b.startObject("my_field2").field("type", "integer").field("store", "false").endObject();
b.startObject("my_field3").field("type", "long").field("doc_values", "false").endObject();
b.startObject("my_field4").field("type", "float").field("index", "false").endObject();
b.startObject("my_field5").field("type", "double").field("store", "true").endObject();
b.startObject("my_field6").field("type", "date").field("doc_values", "false").endObject();
b.startObject("my_field7").field("type", "boolean").field("doc_values", "false").endObject();
}));
MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
parse(newMapper, indexService.mapperService().documentMapperParser(),
XContentFactory.jsonBuilder().startObject().field("my_field2", "foobar").endObject());
});
ParsedDocument doc = newMapper.parse(source(b -> {
b.field("my_field1", 42);
b.field("my_field2", 43);
b.field("my_field3", 44);
b.field("my_field4", 45);
b.field("my_field5", 46);
b.field("my_field6", Instant.now().toEpochMilli());
b.field("my_field7", true);
}));
assertNull(doc.dynamicMappingsUpdate());
MapperParsingException e = expectThrows(MapperParsingException.class,
() -> newMapper.parse(source(b -> b.field("my_field2", "foobar"))));
assertThat(e.getMessage(), containsString("failed to parse field [my_field2] of type [integer]"));
}
public void testMixTemplateMultiFieldAndMappingReuse() throws Exception {
IndexService indexService = createIndex("test");
XContentBuilder mappings1 = jsonBuilder().startObject()
.startObject("_doc")
.startArray("dynamic_templates")
.startObject()
.startObject("template1")
.field("match_mapping_type", "string")
.startObject("mapping")
.field("type", "text")
.startObject("fields")
.startObject("raw")
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endArray()
.endObject().endObject();
indexService.mapperService().merge("_doc", new CompressedXContent(BytesReference.bytes(mappings1)),
MapperService.MergeReason.MAPPING_UPDATE);
MapperService mapperService = createMapperService(topMapping(b -> {
b.startArray("dynamic_templates");
{
b.startObject();
{
b.startObject("template1");
{
b.field("match_mapping_type", "string");
b.startObject("mapping");
{
b.field("type", "text");
b.startObject("fields");
{
b.startObject("raw").field("type", "keyword").endObject();
}
b.endObject();
}
b.endObject();
}
b.endObject();
}
b.endObject();
}
b.endArray();
}));
assertNull(mapperService.documentMapper().mappers().getMapper("field.raw"));
XContentBuilder json = XContentFactory.jsonBuilder().startObject()
.field("field", "foo")
.endObject();
SourceToParse source = new SourceToParse("test", "_doc", "1", BytesReference.bytes(json), json.contentType());
DocumentMapper mapper = indexService.mapperService().documentMapper("_doc");
assertNull(mapper.mappers().getMapper("field.raw"));
ParsedDocument parsed = mapper.parse(source);
ParsedDocument parsed = mapperService.documentMapper().parse(source(b -> b.field("field", "foo")));
assertNotNull(parsed.dynamicMappingsUpdate());
indexService.mapperService().merge("_doc", new CompressedXContent(parsed.dynamicMappingsUpdate().toString()),
MapperService.MergeReason.MAPPING_UPDATE);
mapper = indexService.mapperService().documentMapper("_doc");
assertNotNull(mapper.mappers().getMapper("field.raw"));
parsed = mapper.parse(source);
merge(mapperService, dynamicMapping(parsed.dynamicMappingsUpdate()));
assertNotNull(mapperService.documentMapper().mappers().getMapper("field.raw"));
parsed = mapperService.documentMapper().parse(source(b -> b.field("field", "foo")));
assertNull(parsed.dynamicMappingsUpdate());
}
public void testDefaultFloatingPointMappings() throws IOException {
MapperService mapperService = createIndex("test").mapperService();
String mapping = Strings.toString(jsonBuilder().startObject()
.startObject("type")
.field("numeric_detection", true)
.endObject().endObject());
mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
DocumentMapper mapper = mapperService.documentMapper("type");
DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("numeric_detection", true)));
doTestDefaultFloatingPointMappings(mapper, XContentFactory.jsonBuilder());
doTestDefaultFloatingPointMappings(mapper, XContentFactory.yamlBuilder());
doTestDefaultFloatingPointMappings(mapper, XContentFactory.smileBuilder());
@ -528,7 +344,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
.field("baz", (double) 3.2f) // double that can be accurately represented as a float
.field("quux", "3.2") // float detected through numeric detection
.endObject());
ParsedDocument parsedDocument = mapper.parse(new SourceToParse("index", "type", "id",
ParsedDocument parsedDocument = mapper.parse(new SourceToParse("index", "_doc", "id",
source, builder.contentType()));
Mapping update = parsedDocument.dynamicMappingsUpdate();
assertNotNull(update);
@ -539,107 +355,84 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
}
public void testNumericDetectionEnabled() throws Exception {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.field("numeric_detection", true)
.endObject().endObject());
MapperService mapperService = createMapperService(topMapping(b -> b.field("numeric_detection", true)));
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping, XContentType.JSON).get();
DocumentMapper defaultMapper = index.mapperService().documentMapper("type");
ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("s_long", "100")
.field("s_double", "100.0")
.endObject()),
XContentType.JSON));
ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
b.field("s_long", "100");
b.field("s_double", "100.0");
}));
assertNotNull(doc.dynamicMappingsUpdate());
client().admin().indices().preparePutMapping("test").setType("type")
.setSource(doc.dynamicMappingsUpdate().toString(), XContentType.JSON).get();
merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate()));
defaultMapper = index.mapperService().documentMapper("type");
Mapper mapper = defaultMapper.mappers().getMapper("s_long");
Mapper mapper = mapperService.documentMapper().mappers().getMapper("s_long");
assertThat(mapper.typeName(), equalTo("long"));
mapper = defaultMapper.mappers().getMapper("s_double");
mapper = mapperService.documentMapper().mappers().getMapper("s_double");
assertThat(mapper.typeName(), equalTo("float"));
}
public void testNumericDetectionDefault() throws Exception {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.endObject().endObject());
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping, XContentType.JSON).get();
DocumentMapper defaultMapper = index.mapperService().documentMapper("type");
MapperService mapperService = createMapperService(mapping(b -> {}));
ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("s_long", "100")
.field("s_double", "100.0")
.endObject()),
XContentType.JSON));
ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
b.field("s_long", "100");
b.field("s_double", "100.0");
}));
assertNotNull(doc.dynamicMappingsUpdate());
assertAcked(client().admin().indices().preparePutMapping("test").setType("type")
.setSource(doc.dynamicMappingsUpdate().toString(), XContentType.JSON).get());
merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate()));
defaultMapper = index.mapperService().documentMapper("type");
Mapper mapper = defaultMapper.mappers().getMapper("s_long");
Mapper mapper = mapperService.documentMapper().mappers().getMapper("s_long");
assertThat(mapper, instanceOf(TextFieldMapper.class));
mapper = defaultMapper.mappers().getMapper("s_double");
mapper = mapperService.documentMapper().mappers().getMapper("s_double");
assertThat(mapper, instanceOf(TextFieldMapper.class));
}
public void testDateDetectionInheritsFormat() throws Exception {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startArray("dynamic_date_formats")
.value("yyyy-MM-dd")
.endArray()
.startArray("dynamic_templates")
.startObject()
.startObject("dates")
.field("match_mapping_type", "date")
.field("match", "*2")
.startObject("mapping")
.endObject()
.endObject()
.endObject()
.startObject()
.startObject("dates")
.field("match_mapping_type", "date")
.field("match", "*3")
.startObject("mapping")
.field("format", "yyyy-MM-dd||epoch_millis")
.endObject()
.endObject()
.endObject()
.endArray()
.endObject().endObject());
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping, XContentType.JSON).get();
DocumentMapper defaultMapper = index.mapperService().documentMapper("type");
MapperService mapperService = createMapperService(topMapping(b -> {
b.startArray("dynamic_date_formats").value("yyyy-MM-dd").endArray();
b.startArray("dynamic_templates");
{
b.startObject();
{
b.startObject("dates");
{
b.field("match_mapping_type", "date");
b.field("match", "*2");
b.startObject("mapping").endObject();
}
b.endObject();
}
b.endObject();
b.startObject();
{
b.startObject("dates");
{
b.field("match_mapping_type", "date");
b.field("match", "*3");
b.startObject("mapping").field("format", "yyyy-MM-dd||epoch_millis").endObject();
}
b.endObject();
}
b.endObject();
}
b.endArray();
}));
ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("date1", "2016-11-20")
.field("date2", "2016-11-20")
.field("date3", "2016-11-20")
.endObject()),
XContentType.JSON));
ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
b.field("date1", "2016-11-20");
b.field("date2", "2016-11-20");
b.field("date3", "2016-11-20");
}));
assertNotNull(doc.dynamicMappingsUpdate());
assertAcked(client().admin().indices().preparePutMapping("test").setType("type")
.setSource(doc.dynamicMappingsUpdate().toString(), XContentType.JSON).get());
defaultMapper = index.mapperService().documentMapper("type");
merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate()));
DateFieldMapper dateMapper1 = (DateFieldMapper) defaultMapper.mappers().getMapper("date1");
DateFieldMapper dateMapper2 = (DateFieldMapper) defaultMapper.mappers().getMapper("date2");
DateFieldMapper dateMapper3 = (DateFieldMapper) defaultMapper.mappers().getMapper("date3");
DateFieldMapper dateMapper1 = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("date1");
DateFieldMapper dateMapper2 = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("date2");
DateFieldMapper dateMapper3 = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("date3");
// inherited from dynamic date format
assertEquals("yyyy-MM-dd", dateMapper1.fieldType().dateTimeFormatter().pattern());
// inherited from dynamic date format since the mapping in the template did not specify a format
@ -651,37 +444,36 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
public void testDynamicTemplateOrder() throws IOException {
// https://github.com/elastic/elasticsearch/issues/18625
// elasticsearch used to apply templates that do not have a match_mapping_type first
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startArray("dynamic_templates")
.startObject()
.startObject("type-based")
.field("match_mapping_type", "string")
.startObject("mapping")
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.startObject()
.startObject("path-based")
.field("path_match", "foo")
.startObject("mapping")
.field("type", "long")
.endObject()
.endObject()
.endObject()
.endArray()
.endObject().endObject();
IndexService index = createIndex("test", Settings.EMPTY, "type", mapping);
client().prepareIndex("test", "type", "1").setSource("foo", "abc").get();
assertThat(index.mapperService().fieldType("foo"), instanceOf(KeywordFieldMapper.KeywordFieldType.class));
}
public void testMappingVersionAfterDynamicMappingUpdate() {
createIndex("test", client().admin().indices().prepareCreate("test").addMapping("type"));
final ClusterService clusterService = getInstanceFromNode(ClusterService.class);
final long previousVersion = clusterService.state().metadata().index("test").getMappingVersion();
client().prepareIndex("test", "type", "1").setSource("field", "text").get();
assertThat(clusterService.state().metadata().index("test").getMappingVersion(), equalTo(1 + previousVersion));
MapperService mapperService = createMapperService(topMapping(b -> {
b.startArray("dynamic_templates");
{
b.startObject();
{
b.startObject("type-based");
{
b.field("match_mapping_type", "string");
b.startObject("mapping").field("type", "keyword").endObject();
}
b.endObject();
}
b.endObject();
b.startObject();
{
b.startObject("path-based");
{
b.field("path_match", "foo");
b.startObject("mapping").field("type", "long").endObject();
}
b.endObject();
}
b.endObject();
}
b.endArray();
}));
ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("foo", "abc")));
assertNotNull(doc.dynamicMappingsUpdate());
merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate()));
assertThat(mapperService.fieldType("foo"), instanceOf(KeywordFieldMapper.KeywordFieldType.class));
}
}

71
server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingVersionTests.java
View File

@ -1,71 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
import java.io.IOException;
import java.util.Collection;
public class DynamicMappingVersionTests extends ESSingleNodeTestCase {
@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return pluginList(InternalSettingsPlugin.class);
}
public void testDynamicMappingDefault() throws IOException {
MapperService mapperService = createIndex("my-index").mapperService();
DocumentMapper documentMapper = mapperService
.documentMapperWithAutoCreate("my-type").getDocumentMapper();
ParsedDocument parsedDoc = documentMapper.parse(
new SourceToParse("my-index", "my-type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("foo", 3)
.endObject()), XContentType.JSON));
String expectedMapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("my-type")
.startObject("properties")
.startObject("foo").field("type", "long")
.endObject().endObject().endObject().endObject());
assertEquals(expectedMapping, parsedDoc.dynamicMappingsUpdate().toString());
}
public void testDynamicMappingSettingRemoval() {
Settings settings = Settings.builder()
.put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false)
.build();
Exception e = expectThrows(IllegalArgumentException.class, () -> createIndex("test-index", settings));
assertEquals(e.getMessage(), "Setting index.mapper.dynamic was removed after version 6.0.0");
assertSettingDeprecationsAndWarnings(new Setting[] { MapperService.INDEX_MAPPER_DYNAMIC_SETTING });
}
}

97
server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
View File

@ -22,38 +22,37 @@ package org.elasticsearch.index.mapper;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.test.ESSingleNodeTestCase;
import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
public class DynamicTemplatesTests extends ESSingleNodeTestCase {
public void testMatchTypeOnly() throws Exception {
XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject().startObject("person").startArray("dynamic_templates").startObject().startObject("test")
.field("match_mapping_type", "string")
.startObject("mapping").field("index", false).endObject()
.endObject().endObject().endArray().endObject().endObject();
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("person").setSource(builder).get();
public class DynamicTemplatesTests extends MapperServiceTestCase {
MapperService mapperService = index.mapperService();
DocumentMapper docMapper = mapperService.documentMapper("person");
builder = JsonXContent.contentBuilder();
builder.startObject().field("s", "hello").field("l", 1).endObject();
ParsedDocument parsedDoc = docMapper.parse(new SourceToParse("test", "person", "1", BytesReference.bytes(builder),
XContentType.JSON));
client().admin().indices().preparePutMapping("test").setType("person")
.setSource(parsedDoc.dynamicMappingsUpdate().toString(), XContentType.JSON).get();
public void testMatchTypeOnly() throws Exception {
MapperService mapperService = createMapperService(topMapping(b -> {
b.startArray("dynamic_templates");
{
b.startObject();
{
b.startObject("test");
{
b.field("match_mapping_type", "string");
b.startObject("mapping").field("index", false).endObject();
}
b.endObject();
}
b.endObject();
}
b.endArray();
}));
DocumentMapper docMapper = mapperService.documentMapper();
ParsedDocument parsedDoc = docMapper.parse(source(b -> {
b.field("s", "hello");
b.field("l", 1);
}));
merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate()));
assertThat(mapperService.fieldType("s"), notNullValue());
assertFalse(mapperService.fieldType("s").isSearchable());
@ -64,15 +63,11 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
public void testSimple() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-mapping.json");
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping, XContentType.JSON).get();
DocumentMapper docMapper = index.mapperService().documentMapper("person");
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-data.json");
ParsedDocument parsedDoc = docMapper.parse(new SourceToParse("test", "person", "1", new BytesArray(json),
XContentType.JSON));
client().admin().indices().preparePutMapping("test").setType("person")
.setSource(parsedDoc.dynamicMappingsUpdate().toString(), XContentType.JSON).get();
docMapper = index.mapperService().documentMapper("person");
MapperService mapperService = createMapperService("person", mapping);
String docJson = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-data.json");
ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson));
merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate()));
Document doc = parsedDoc.rootDoc();
IndexableField f = doc.getField("name");
@ -81,7 +76,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
assertThat(f.fieldType().tokenized(), equalTo(false));
Mapper fieldMapper = docMapper.mappers().getMapper("name");
Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("name");
assertNotNull(fieldMapper);
f = doc.getField("multi1");
@ -90,7 +85,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
assertThat(f.fieldType().tokenized(), equalTo(true));
fieldMapper = docMapper.mappers().getMapper("multi1");
fieldMapper = mapperService.documentMapper().mappers().getMapper("multi1");
assertNotNull(fieldMapper);
f = doc.getField("multi1.org");
@ -99,7 +94,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
assertThat(f.fieldType().tokenized(), equalTo(false));
fieldMapper = docMapper.mappers().getMapper("multi1.org");
fieldMapper = mapperService.documentMapper().mappers().getMapper("multi1.org");
assertNotNull(fieldMapper);
f = doc.getField("multi2");
@ -108,7 +103,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
assertThat(f.fieldType().tokenized(), equalTo(true));
fieldMapper = docMapper.mappers().getMapper("multi2");
fieldMapper = mapperService.documentMapper().mappers().getMapper("multi2");
assertNotNull(fieldMapper);
f = doc.getField("multi2.org");
@ -117,21 +112,17 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
assertThat(f.fieldType().tokenized(), equalTo(false));
fieldMapper = docMapper.mappers().getMapper("multi2.org");
fieldMapper = mapperService.documentMapper().mappers().getMapper("multi2.org");
assertNotNull(fieldMapper);
}
public void testSimpleWithXContentTraverse() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-mapping.json");
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping, XContentType.JSON).get();
DocumentMapper docMapper = index.mapperService().documentMapper("person");
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-data.json");
ParsedDocument parsedDoc = docMapper.parse(new SourceToParse("test", "person", "1", new BytesArray(json),
XContentType.JSON));
client().admin().indices().preparePutMapping("test").setType("person")
.setSource(parsedDoc.dynamicMappingsUpdate().toString(), XContentType.JSON).get();
docMapper = index.mapperService().documentMapper("person");
MapperService mapperService = createMapperService("person", mapping);
String docJson = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-data.json");
ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson));
merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate()));
Document doc = parsedDoc.rootDoc();
IndexableField f = doc.getField("name");
@ -140,7 +131,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
assertThat(f.fieldType().tokenized(), equalTo(false));
Mapper fieldMapper = docMapper.mappers().getMapper("name");
Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("name");
assertNotNull(fieldMapper);
f = doc.getField("multi1");
@ -149,7 +140,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
assertThat(f.fieldType().tokenized(), equalTo(true));
fieldMapper = docMapper.mappers().getMapper("multi1");
fieldMapper = mapperService.documentMapper().mappers().getMapper("multi1");
assertNotNull(fieldMapper);
f = doc.getField("multi1.org");
@ -158,7 +149,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
assertThat(f.fieldType().tokenized(), equalTo(false));
fieldMapper = docMapper.mappers().getMapper("multi1.org");
fieldMapper = mapperService.documentMapper().mappers().getMapper("multi1.org");
assertNotNull(fieldMapper);
f = doc.getField("multi2");
@ -167,7 +158,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
assertThat(f.fieldType().tokenized(), equalTo(true));
fieldMapper = docMapper.mappers().getMapper("multi2");
fieldMapper = mapperService.documentMapper().mappers().getMapper("multi2");
assertNotNull(fieldMapper);
f = doc.getField("multi2.org");
@ -176,7 +167,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
assertThat(f.fieldType().tokenized(), equalTo(false));
fieldMapper = docMapper.mappers().getMapper("multi2.org");
fieldMapper = mapperService.documentMapper().mappers().getMapper("multi2.org");
assertNotNull(fieldMapper);
}
}

181
server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
View File

@ -21,77 +21,28 @@ package org.elasticsearch.index.mapper;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.indices.mapper.MapperRegistry;
import org.elasticsearch.plugins.MapperPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
import org.elasticsearch.test.VersionUtils;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Supplier;
import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.closeTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
public class ExternalFieldMapperTests extends ESSingleNodeTestCase {
public class ExternalFieldMapperTests extends MapperServiceTestCase {
@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return pluginList(InternalSettingsPlugin.class);
}
@Override
protected boolean forbidPrivateIndexSettings() {
return false;
protected Collection<? extends Plugin> getPlugins() {
return Collections.singletonList(new ExternalMapperPlugin());
}
public void testExternalValues() throws Exception {
Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0,
Version.CURRENT);
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
IndexService indexService = createIndex("test", settings);
MapperRegistry mapperRegistry = new MapperRegistry(
singletonMap(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")),
singletonMap(ExternalMetadataMapper.CONTENT_TYPE, ExternalMetadataMapper.PARSER), MapperPlugin.NOOP_FIELD_FILTER);
Supplier<QueryShardContext> queryShardContext = () -> {
return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null);
};
DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null);
DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(
Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject(ExternalMetadataMapper.CONTENT_TYPE)
.endObject()
.startObject("properties")
.startObject("field").field("type", "external").endObject()
.endObject()
.endObject().endObject())
));
DocumentMapper documentMapper = createDocumentMapper(fieldMapping(b -> b.field("type", "external")));
ParsedDocument doc = documentMapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", "1234")
.endObject()),
XContentType.JSON));
ParsedDocument doc = documentMapper.parse(source(b -> b.field("field", "1234")));
assertThat(doc.rootDoc().getField("field.bool"), notNullValue());
assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T"));
@ -111,45 +62,32 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase {
}
public void testExternalValuesWithMultifield() throws Exception {
IndexService indexService = createIndex("test");
Map<String, Mapper.TypeParser> mapperParsers = new HashMap<>();
mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo"));
mapperParsers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser());
mapperParsers.put(KeywordFieldMapper.CONTENT_TYPE, KeywordFieldMapper.PARSER);
MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER);
Supplier<QueryShardContext> queryShardContext = () -> {
return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null);
};
DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null);
DocumentMapper documentMapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", ExternalMapperPlugin.EXTERNAL);
b.startObject("fields");
{
b.startObject("text");
{
b.field("type", "text");
b.field("store", true);
b.startObject("fields");
{
b.startObject("raw");
{
b.field("type", "keyword");
b.field("store", true);
}
b.endObject();
}
b.endObject();
}
b.endObject();
}
b.endObject();
}));
DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(
Strings
.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("field")
.field("type", ExternalMapperPlugin.EXTERNAL)
.startObject("fields")
.startObject("text")
.field("type", "text")
.field("store", true)
.startObject("fields")
.startObject("raw")
.field("type", "keyword")
.field("store", true)
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject().endObject().endObject())));
ParsedDocument doc = documentMapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", "1234")
.endObject()),
XContentType.JSON));
ParsedDocument doc = documentMapper.parse(source(b -> b.field("field", "1234")));
assertThat(doc.rootDoc().getField("field.bool"), notNullValue());
assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T"));
@ -179,49 +117,28 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase {
}
public void testExternalValuesWithMultifieldTwoLevels() throws Exception {
IndexService indexService = createIndex("test");
Map<String, Mapper.TypeParser> mapperParsers = new HashMap<>();
mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo"));
mapperParsers.put(ExternalMapperPlugin.EXTERNAL_BIS, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "bar"));
mapperParsers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser());
MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER);
Supplier<QueryShardContext> queryShardContext = () -> {
return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null);
};
DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null);
DocumentMapper documentMapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", ExternalMapperPlugin.EXTERNAL);
b.startObject("fields");
{
b.startObject("text");
{
b.field("type", "text");
b.startObject("fields");
{
b.startObject("generated").field("type", ExternalMapperPlugin.EXTERNAL_BIS).endObject();
b.startObject("raw").field("type", "text").endObject();
}
b.endObject();
}
b.endObject();
b.startObject("raw").field("type", "text").endObject();
}
b.endObject();
}));
DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(
Strings
.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("field")
.field("type", ExternalMapperPlugin.EXTERNAL)
.startObject("fields")
.startObject("text")
.field("type", "text")
.startObject("fields")
.startObject("generated")
.field("type", ExternalMapperPlugin.EXTERNAL_BIS)
.endObject()
.startObject("raw")
.field("type", "text")
.endObject()
.endObject()
.endObject()
.startObject("raw")
.field("type", "text")
.endObject()
.endObject()
.endObject()
.endObject().endObject().endObject())));
ParsedDocument doc = documentMapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", "1234")
.endObject()),
XContentType.JSON));
ParsedDocument doc = documentMapper.parse(source(b -> b.field("field", "1234")));
assertThat(doc.rootDoc().getField("field.bool"), notNullValue());
assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T"));

153
server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperTests.java
View File

@ -20,30 +20,16 @@
package org.elasticsearch.index.mapper;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.MapperService.MergeReason;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.junit.Before;
import java.io.IOException;
public class FieldAliasMapperTests extends ESSingleNodeTestCase {
private MapperService mapperService;
private DocumentMapperParser parser;
@Before
public void setup() {
IndexService indexService = createIndex("test");
mapperService = indexService.mapperService();
parser = mapperService.documentMapperParser();
}
public class FieldAliasMapperTests extends MapperServiceTestCase {
public void testParsing() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("_doc")
.startObject("properties")
.startObject("alias-field")
.field("type", "alias")
@ -55,79 +41,56 @@ public class FieldAliasMapperTests extends ESSingleNodeTestCase {
.endObject()
.endObject()
.endObject());
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
DocumentMapper mapper = createDocumentMapper("_doc", mapping);
assertEquals(mapping, mapper.mappingSource().toString());
}
public void testParsingWithMissingPath() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("alias-field")
.field("type", "alias")
.endObject()
.endObject()
.endObject()
.endObject());
public void testParsingWithMissingPath() {
MapperParsingException exception = expectThrows(MapperParsingException.class,
() -> parser.parse("type", new CompressedXContent(mapping)));
assertEquals("The [path] property must be specified for field [alias-field].", exception.getMessage());
() -> createDocumentMapper(mapping(b -> b.startObject("alias-field").field("type", "alias").endObject())));
assertEquals("Failed to parse mapping [_doc]: The [path] property must be specified for field [alias-field].",
exception.getMessage());
}
public void testParsingWithExtraArgument() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("alias-field")
.field("type", "alias")
.field("path", "concrete-field")
.field("extra-field", "extra-value")
.endObject()
.endObject()
.endObject()
.endObject());
MapperParsingException exception = expectThrows(MapperParsingException.class,
() -> parser.parse("type", new CompressedXContent(mapping)));
assertEquals("Mapping definition for [alias-field] has unsupported parameters: [extra-field : extra-value]",
public void testParsingWithExtraArgument() {
MapperParsingException exception = expectThrows(MapperParsingException.class, () -> createDocumentMapper(mapping(b -> {
b.startObject("alias-field");
{
b.field("type", "alias");
b.field("path", "concrete-field");
b.field("extra-field", "extra-value");
}
b.endObject();
})));
assertEquals("Failed to parse mapping [_doc]: " +
"Mapping definition for [alias-field] has unsupported parameters: [extra-field : extra-value]",
exception.getMessage());
}
public void testMerge() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("properties")
.startObject("first-field")
.field("type", "keyword")
.endObject()
.startObject("alias-field")
.field("type", "alias")
.field("path", "first-field")
.endObject()
.endObject()
.endObject()
.endObject());
mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
MapperService mapperService = createMapperService(mapping(b -> {
b.startObject("first-field").field("type", "keyword").endObject();
b.startObject("alias-field");
{
b.field("type", "alias");
b.field("path", "first-field");
}
b.endObject();
}));
MappedFieldType firstFieldType = mapperService.fieldType("alias-field");
assertEquals("first-field", firstFieldType.name());
assertTrue(firstFieldType instanceof KeywordFieldMapper.KeywordFieldType);
String newMapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("properties")
.startObject("second-field")
.field("type", "text")
.endObject()
.startObject("alias-field")
.field("type", "alias")
.field("path", "second-field")
.endObject()
.endObject()
.endObject()
.endObject());
mapperService.merge("type", new CompressedXContent(newMapping), MergeReason.MAPPING_UPDATE);
merge(mapperService, mapping(b -> {
b.startObject("second-field").field("type", "text").endObject();
b.startObject("alias-field");
{
b.field("type", "alias");
b.field("path", "second-field");
}
b.endObject();
}));
MappedFieldType secondFieldType = mapperService.fieldType("alias-field");
assertEquals("second-field", secondFieldType.name());
@ -135,32 +98,24 @@ public class FieldAliasMapperTests extends ESSingleNodeTestCase {
}
public void testMergeFailure() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("properties")
.startObject("concrete-field")
.field("type", "text")
.endObject()
.startObject("alias-field")
.field("type", "alias")
.field("path", "concrete-field")
.endObject()
.endObject()
.endObject()
.endObject());
mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
String newMapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("properties")
.startObject("alias-field")
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.endObject());
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
() -> mapperService.merge("type", new CompressedXContent(newMapping), MergeReason.MAPPING_UPDATE));
MapperService mapperService = createMapperService(mapping(b -> {
b.startObject("concrete-field").field("type", "text").endObject();
b.startObject("alias-field");
{
b.field("type", "alias");
b.field("path", "concrete-field");
}
b.endObject();
}));
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> merge(mapperService, mapping(b -> {
b.startObject("alias-field");
{
b.field("type", "keyword");
}
b.endObject();
})));
assertEquals("Cannot merge a field alias mapping [alias-field] with a mapping that is not for a field alias.",
exception.getMessage());
}

11
test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
View File

@ -27,6 +27,7 @@ import org.apache.lucene.store.Directory;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
@ -104,6 +105,12 @@ public abstract class MapperServiceTestCase extends ESTestCase {
return createMapperService(getIndexSettings(), mappings);
}
protected final MapperService createMapperService(String type, String mappings) throws IOException {
MapperService mapperService = createMapperService(mapping(b -> {}));
merge(type, mapperService, mappings);
return mapperService;
}
/**
* Create a {@link MapperService} like we would for an index.
*/
@ -160,6 +167,10 @@ public abstract class MapperServiceTestCase extends ESTestCase {
return new SourceToParse("test", "_doc", "1", BytesReference.bytes(builder), XContentType.JSON);
}
protected final SourceToParse source(String source) {
return new SourceToParse("test", "_doc", "1", new BytesArray(source), XContentType.JSON);
}
/**
* Merge a new mapping into the one in the provided {@link MapperService}.
*/