honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Mappings: Enforce metadata fields are not passed in documents 

We previously removed the ability to specify metadata fields inside
documents in #11074, but the backcompat left leniency that allowed this
to still occur. This change locks down parsing so any metadata field
found while parsing a document results in an exception. This only
affects 2.0+ indexes; backcompat is maintained.

closes #13740
This commit is contained in:
Ryan Ernst 2015-10-07 12:52:13 -07:00
parent c0363dd56b
commit 566fef0cde
8 changed files with 89 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View File

@ -122,7 +122,7 @@ class DocumentParser implements Closeable {
// entire type is disabled
parser.skipChildren();
} else if (emptyDoc == false) {
Mapper update = parseObject(context, mapping.root);
Mapper update = parseObject(context, mapping.root, true);
if (update != null) {
context.addDynamicMappingsUpdate(update);
}
@ -194,7 +194,7 @@ class DocumentParser implements Closeable {
return doc;
}
static ObjectMapper parseObject(ParseContext context, ObjectMapper mapper) throws IOException {
static ObjectMapper parseObject(ParseContext context, ObjectMapper mapper, boolean atRoot) throws IOException {
if (mapper.isEnabled() == false) {
context.parser().skipChildren();
return null;
@ -202,6 +202,10 @@ class DocumentParser implements Closeable {
XContentParser parser = context.parser();
String currentFieldName = parser.currentName();
if (atRoot && MapperService.isMetadataField(currentFieldName) &&
Version.indexCreated(context.indexSettings()).onOrAfter(Version.V_2_0_0_beta1)) {
throw new MapperParsingException("Field [" + currentFieldName + "] is a metadata field and cannot be added inside a document. Use the index API request parameters.");
}
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
// the object is null ("obj1" : null), simply bail
@ -302,7 +306,7 @@ class DocumentParser implements Closeable {
private static Mapper parseObjectOrField(ParseContext context, Mapper mapper) throws IOException {
if (mapper instanceof ObjectMapper) {
return parseObject(context, (ObjectMapper) mapper);
return parseObject(context, (ObjectMapper) mapper, false);
} else {
FieldMapper fieldMapper = (FieldMapper)mapper;
Mapper update = fieldMapper.parse(context);

2
core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
View File

@ -197,7 +197,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
ctx.reset(XContentHelper.createParser(source.source()), new ParseContext.Document(), source);
assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
ctx.parser().nextToken();
return DocumentParser.parseObject(ctx, mapper.root());
return DocumentParser.parseObject(ctx, mapper.root(), true);
}
public void testDynamicMappingsNotNeeded() throws Exception {

14
core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
View File

@ -43,6 +43,7 @@ import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
@ -453,4 +454,17 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase {
// the backcompat behavior is actually ignoring directly specifying _all
assertFalse(field.getAllEntries().fields().iterator().hasNext());
}
public void testIncludeInObjectNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
try {
docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject().field("_all", "foo").endObject().bytes());
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_all] is a metadata field and cannot be added inside a document"));
}
}
}

13
core/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java
View File

@ -114,4 +114,17 @@ public class IdMappingTests extends ESSingleNodeTestCase {
// _id is not indexed so we need to check _uid
assertEquals(Uid.createUid("type", "1"), doc.rootDoc().get(UidFieldMapper.NAME));
}
public void testIncludeInObjectNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
try {
docMapper.parse(SourceToParse.source(XContentFactory.jsonBuilder()
.startObject().field("_id", "1").endObject().bytes()).type("type"));
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document"));
}
}
}

20
core/src/test/java/org/elasticsearch/index/mapper/parent/ParentMappingTests.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.Uid;
@ -32,21 +33,18 @@ import static org.hamcrest.Matchers.nullValue;
public class ParentMappingTests extends ESSingleNodeTestCase {
public void testParentNotSet() throws Exception {
public void testParentSetInDocNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
ParsedDocument doc = docMapper.parse(SourceToParse.source(XContentFactory.jsonBuilder()
.startObject()
.field("_parent", "1122")
.field("x_field", "x_value")
.endObject()
.bytes()).type("type").id("1"));
// no _parent mapping, dynamically used as a string field
assertNull(doc.parent());
assertNotNull(doc.rootDoc().get("_parent"));
try {
docMapper.parse(SourceToParse.source(XContentFactory.jsonBuilder()
.startObject().field("_parent", "1122").endObject().bytes()).type("type").id("1"));
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_parent] is a metadata field and cannot be added inside a document"));
}
}
public void testParentSetInDocBackcompat() throws Exception {

16
core/src/test/java/org/elasticsearch/index/mapper/routing/RoutingTypeMapperTests.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.test.ESSingleNodeTestCase;
@ -113,7 +114,7 @@ public class RoutingTypeMapperTests extends ESSingleNodeTestCase {
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
DocumentMapper docMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("_timestamp", 2000000).endObject();
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("_routing", "foo").endObject();
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(MetaData.builder().build(), mappingMetaData, true, "test");
@ -122,4 +123,17 @@ public class RoutingTypeMapperTests extends ESSingleNodeTestCase {
assertNull(request.routing());
assertNull(docMapper.parse("test", "type", "1", doc.bytes()).rootDoc().get("_routing"));
}
public void testIncludeInObjectNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
try {
docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject().field("_routing", "foo").endObject().bytes());
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_routing] is a metadata field and cannot be added inside a document"));
}
}
}

15
core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
View File

@ -769,6 +769,21 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
assertNull(docMapper.parse("test", "type", "1", doc.bytes()).rootDoc().get("_timestamp"));
}
public void testIncludeInObjectNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", true).field("default", "1970").field("format", "YYYY").endObject()
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
try {
docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject().field("_timestamp", 2000000).endObject().bytes());
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_timestamp] is a metadata field and cannot be added inside a document"));
}
}
public void testThatEpochCanBeIgnoredWithCustomFormat() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject()

15
core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
View File

@ -310,6 +310,21 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
assertNull(docMapper.parse("test", "type", "1", doc.bytes()).rootDoc().get("_ttl"));
}
public void testIncludeInObjectNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_ttl").field("enabled", true).endObject()
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
try {
docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject().field("_ttl", "2d").endObject().bytes());
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_ttl] is a metadata field and cannot be added inside a document"));
}
}
private org.elasticsearch.common.xcontent.XContentBuilder getMappingWithTtlEnabled() throws IOException {
return getMappingWithTtlEnabled(null);
}