honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Mapping - MultiField Mapping, closes #51.

This commit is contained in:
kimchy 2010-03-07 20:13:59 +02:00
parent 54dc5a59b1
commit b13f6b1bdd
13 changed files with 517 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDocumentMapperParser.java
View File

@ -261,6 +261,8 @@ public class JsonDocumentMapperParser implements DocumentMapperParser {
// lets see if we can derive this...
if (propNode.isObject() && propNode.get("properties") != null) {
type = JsonObjectMapper.JSON_TYPE;
} else if (propNode.isObject() && propNode.get("fields") != null) {
type = JsonMultiFieldMapper.JSON_TYPE;
} else {
throw new MapperParsingException("No type specified for property [" + propName + "]");
}
@ -281,12 +283,60 @@ public class JsonDocumentMapperParser implements DocumentMapperParser {
objBuilder.add(parseBoolean(propName, (ObjectNode) propNode));
} else if (type.equals(JsonObjectMapper.JSON_TYPE)) {
objBuilder.add(parseObject(propName, (ObjectNode) propNode));
} else if (type.equals(JsonMultiFieldMapper.JSON_TYPE)) {
objBuilder.add(parseMultiField(propName, (ObjectNode) propNode));
} else if (type.equals(JsonBinaryFieldMapper.JSON_TYPE)) {
objBuilder.add(parseBinary(propName, (ObjectNode) propNode));
}
}
}
private JsonMultiFieldMapper.Builder parseMultiField(String name, ObjectNode multiFieldNode) {
JsonMultiFieldMapper.Builder builder = multiField(name);
for (Iterator<Map.Entry<String, JsonNode>> fieldsIt = multiFieldNode.getFields(); fieldsIt.hasNext();) {
Map.Entry<String, JsonNode> entry = fieldsIt.next();
String fieldName = entry.getKey();
JsonNode fieldNode = entry.getValue();
if (fieldName.equals("pathType")) {
builder.pathType(parsePathType(name, fieldNode.getValueAsText()));
} else if (fieldName.equals("fields")) {
ObjectNode fieldsNode = (ObjectNode) fieldNode;
for (Iterator<Map.Entry<String, JsonNode>> propsIt = fieldsNode.getFields(); propsIt.hasNext();) {
Map.Entry<String, JsonNode> entry1 = propsIt.next();
String propName = entry1.getKey();
JsonNode propNode = entry1.getValue();
String type;
JsonNode typeNode = propNode.get("type");
if (typeNode != null) {
type = typeNode.getTextValue();
} else {
throw new MapperParsingException("No type specified for property [" + propName + "]");
}
if (type.equals(JsonStringFieldMapper.JSON_TYPE)) {
builder.add(parseString(propName, (ObjectNode) propNode));
} else if (type.equals(JsonDateFieldMapper.JSON_TYPE)) {
builder.add(parseDate(propName, (ObjectNode) propNode));
} else if (type.equals(JsonIntegerFieldMapper.JSON_TYPE)) {
builder.add(parseInteger(propName, (ObjectNode) propNode));
} else if (type.equals(JsonLongFieldMapper.JSON_TYPE)) {
builder.add(parseLong(propName, (ObjectNode) propNode));
} else if (type.equals(JsonFloatFieldMapper.JSON_TYPE)) {
builder.add(parseFloat(propName, (ObjectNode) propNode));
} else if (type.equals(JsonDoubleFieldMapper.JSON_TYPE)) {
builder.add(parseDouble(propName, (ObjectNode) propNode));
} else if (type.equals(JsonBooleanFieldMapper.JSON_TYPE)) {
builder.add(parseBoolean(propName, (ObjectNode) propNode));
} else if (type.equals(JsonBinaryFieldMapper.JSON_TYPE)) {
builder.add(parseBinary(propName, (ObjectNode) propNode));
}
}
}
}
return builder;
}
private JsonDateFieldMapper.Builder parseDate(String name, ObjectNode dateNode) {
JsonDateFieldMapper.Builder builder = dateField(name);
parseNumberField(builder, name, dateNode);

1
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonFieldMapper.java
View File

@ -112,7 +112,6 @@ public abstract class JsonFieldMapper<T> implements FieldMapper<T>, JsonMapper {
protected Builder(String name) {
super(name);
indexName = name;
}
protected T index(Field.Index index) {

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMapper.java
View File

@ -49,7 +49,7 @@ public interface JsonMapper extends ToJson {
@NotThreadSafe
public static abstract class Builder<T extends Builder, Y extends JsonMapper> {
protected final String name;
protected String name;
protected T builder;

4
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMapperBuilders.java
View File

@ -52,6 +52,10 @@ public final class JsonMapperBuilders {
return new JsonBoostFieldMapper.Builder(name);
}
public static JsonMultiFieldMapper.Builder multiField(String name) {
return new JsonMultiFieldMapper.Builder(name);
}
public static JsonObjectMapper.Builder object(String name) {
return new JsonObjectMapper.Builder(name);
}

213
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMultiFieldMapper.java Normal file
View File

@ -0,0 +1,213 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.json;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapperListener;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.util.json.JsonBuilder;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.google.common.collect.Lists.*;
import static org.elasticsearch.util.MapBuilder.*;
/**
* @author kimchy (shay.banon)
*/
public class JsonMultiFieldMapper implements JsonMapper {
public static final String JSON_TYPE = "multi_field";
public static class Defaults {
public static final JsonPath.Type PATH_TYPE = JsonPath.Type.FULL;
}
public static class Builder extends JsonMapper.Builder<Builder, JsonMultiFieldMapper> {
private JsonPath.Type pathType = Defaults.PATH_TYPE;
private final List<JsonMapper.Builder> mappersBuilders = newArrayList();
private JsonMapper.Builder defaultMapperBuilder;
public Builder(String name) {
super(name);
this.builder = this;
}
public Builder pathType(JsonPath.Type pathType) {
this.pathType = pathType;
return this;
}
public Builder add(JsonMapper.Builder builder) {
if (builder.name.equals(name)) {
defaultMapperBuilder = builder;
} else {
mappersBuilders.add(builder);
}
return this;
}
@Override public JsonMultiFieldMapper build(BuilderContext context) {
JsonPath.Type origPathType = context.path().pathType();
context.path().pathType(pathType);
JsonMapper defaultMapper = null;
if (defaultMapperBuilder != null) {
defaultMapper = defaultMapperBuilder.build(context);
}
context.path().add(name);
Map<String, JsonMapper> mappers = new HashMap<String, JsonMapper>();
for (JsonMapper.Builder builder : mappersBuilders) {
JsonMapper mapper = builder.build(context);
mappers.put(mapper.name(), mapper);
}
context.path().remove();
context.path().pathType(origPathType);
return new JsonMultiFieldMapper(name, pathType, mappers, defaultMapper);
}
}
private final String name;
private final JsonPath.Type pathType;
private final Object mutex = new Object();
private volatile ImmutableMap<String, JsonMapper> mappers = ImmutableMap.of();
private volatile JsonMapper defaultMapper;
public JsonMultiFieldMapper(String name, JsonPath.Type pathType, JsonMapper defaultMapper) {
this(name, pathType, new HashMap<String, JsonMapper>(), defaultMapper);
}
public JsonMultiFieldMapper(String name, JsonPath.Type pathType, Map<String, JsonMapper> mappers, JsonMapper defaultMapper) {
this.name = name;
this.pathType = pathType;
this.mappers = ImmutableMap.copyOf(mappers);
this.defaultMapper = defaultMapper;
}
@Override public String name() {
return this.name;
}
public JsonPath.Type pathType() {
return pathType;
}
public JsonMapper defaultMapper() {
return this.defaultMapper;
}
public ImmutableMap<String, JsonMapper> mappers() {
return this.mappers;
}
@Override public void parse(JsonParseContext jsonContext) throws IOException {
JsonPath.Type origPathType = jsonContext.path().pathType();
jsonContext.path().pathType(pathType);
// do the default mapper without adding the path
if (defaultMapper != null) {
defaultMapper.parse(jsonContext);
}
jsonContext.path().add(name);
for (JsonMapper mapper : mappers.values()) {
mapper.parse(jsonContext);
}
jsonContext.path().remove();
jsonContext.path().pathType(origPathType);
}
@Override public void merge(JsonMapper mergeWith, JsonMergeContext mergeContext) throws MergeMappingException {
if (!(mergeWith instanceof JsonMultiFieldMapper)) {
mergeContext.addConflict("Can't merge a non multi_field mapping [" + mergeWith.name() + "] with a multi_field mapping [" + name() + "]");
return;
}
JsonMultiFieldMapper mergeWithMultiField = (JsonMultiFieldMapper) mergeWith;
synchronized (mutex) {
// merge the default mapper
if (defaultMapper == null) {
if (mergeWithMultiField.defaultMapper != null) {
if (!mergeContext.mergeFlags().simulate()) {
defaultMapper = mergeWithMultiField.defaultMapper;
mergeContext.docMapper().addFieldMapper((FieldMapper) defaultMapper);
}
}
} else {
if (mergeWithMultiField.defaultMapper != null) {
defaultMapper.merge(mergeWithMultiField.defaultMapper, mergeContext);
}
}
// merge all the other mappers
for (JsonMapper mergeWithMapper : mergeWithMultiField.mappers.values()) {
JsonMapper mergeIntoMapper = mappers.get(mergeWithMapper.name());
if (mergeIntoMapper == null) {
// no mapping, simply add it if not simulating
if (!mergeContext.mergeFlags().simulate()) {
mappers = newMapBuilder(mappers).put(mergeWithMapper.name(), mergeWithMapper).immutableMap();
if (mergeWithMapper instanceof JsonFieldMapper) {
mergeContext.docMapper().addFieldMapper((FieldMapper) mergeWithMapper);
}
}
} else {
mergeIntoMapper.merge(mergeWithMapper, mergeContext);
}
}
}
}
@Override public void traverse(FieldMapperListener fieldMapperListener) {
for (JsonMapper mapper : mappers.values()) {
mapper.traverse(fieldMapperListener);
}
}
@Override public void toJson(JsonBuilder builder, Params params) throws IOException {
builder.startObject(name);
builder.field("type", JSON_TYPE);
builder.field("pathType", pathType.name().toLowerCase());
builder.startObject("fields");
if (defaultMapper != null) {
defaultMapper.toJson(builder, params);
}
for (JsonMapper mapper : mappers.values()) {
mapper.toJson(builder, params);
}
builder.endObject();
builder.endObject();
}
}

17
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonObjectMapper.java
View File

@ -70,6 +70,7 @@ public class JsonObjectMapper implements JsonMapper {
public Builder(String name) {
super(name);
this.builder = this;
}
public Builder enabled(boolean enabled) {
@ -358,7 +359,21 @@ public class JsonObjectMapper implements JsonMapper {
}
}
} else {
mergeIntoMapper.merge(mergeWithMapper, mergeContext);
if ((mergeWithMapper instanceof JsonMultiFieldMapper) && !(mergeIntoMapper instanceof JsonMultiFieldMapper)) {
JsonMultiFieldMapper mergeWithMultiField = (JsonMultiFieldMapper) mergeWithMapper;
mergeWithMultiField.merge(mergeIntoMapper, mergeContext);
if (!mergeContext.mergeFlags().simulate()) {
putMapper(mergeWithMultiField);
// now, raise events for all mappers
for (JsonMapper mapper : mergeWithMultiField.mappers().values()) {
if (mapper instanceof JsonFieldMapper) {
mergeContext.docMapper().addFieldMapper((FieldMapper) mapper);
}
}
}
} else {
mergeIntoMapper.merge(mergeWithMapper, mergeContext);
}
}
}
}

101
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/multifield/JsonMultiFieldTests.java Normal file
View File

@ -0,0 +1,101 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.json.multifield;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.mapper.json.JsonDocumentMapper;
import org.elasticsearch.index.mapper.json.JsonDocumentMapperParser;
import org.testng.annotations.Test;
import static org.elasticsearch.index.mapper.json.JsonMapperBuilders.*;
import static org.elasticsearch.util.io.Streams.*;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/**
* @author kimchy (shay.banon)
*/
@Test
public class JsonMultiFieldTests {
@Test public void testMultiField() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/json/multifield/test-mapping.json");
JsonDocumentMapper docMapper = (JsonDocumentMapper) new JsonDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/json/multifield/test-data.json");
Document doc = docMapper.parse(json).doc();
Field f = doc.getField("name");
assertThat(f.name(), equalTo("name"));
assertThat(f.stringValue(), equalTo("some name"));
assertThat(f.isStored(), equalTo(true));
assertThat(f.isIndexed(), equalTo(true));
f = doc.getField("name.indexed");
assertThat(f.name(), equalTo("name.indexed"));
assertThat(f.stringValue(), equalTo("some name"));
assertThat(f.isStored(), equalTo(false));
assertThat(f.isIndexed(), equalTo(true));
f = doc.getField("name.not_indexed");
assertThat(f.name(), equalTo("name.not_indexed"));
assertThat(f.stringValue(), equalTo("some name"));
assertThat(f.isStored(), equalTo(true));
assertThat(f.isIndexed(), equalTo(false));
}
@Test public void testBuildThenParse() throws Exception {
JsonDocumentMapper builderDocMapper = doc(object("person").add(
multiField("name")
.add(stringField("name").store(Field.Store.YES))
.add(stringField("indexed").index(Field.Index.ANALYZED))
.add(stringField("not_indexed").index(Field.Index.NO).store(Field.Store.YES))
)).build();
String builtMapping = builderDocMapper.buildSource();
// System.out.println(builtMapping);
// reparse it
JsonDocumentMapper docMapper = (JsonDocumentMapper) new JsonDocumentMapperParser(new AnalysisService(new Index("test"))).parse(builtMapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/json/multifield/test-data.json");
Document doc = docMapper.parse(json).doc();
Field f = doc.getField("name");
assertThat(f.name(), equalTo("name"));
assertThat(f.stringValue(), equalTo("some name"));
assertThat(f.isStored(), equalTo(true));
assertThat(f.isIndexed(), equalTo(true));
f = doc.getField("name.indexed");
assertThat(f.name(), equalTo("name.indexed"));
assertThat(f.stringValue(), equalTo("some name"));
assertThat(f.isStored(), equalTo(false));
assertThat(f.isIndexed(), equalTo(true));
f = doc.getField("name.not_indexed");
assertThat(f.name(), equalTo("name.not_indexed"));
assertThat(f.stringValue(), equalTo("some name"));
assertThat(f.isStored(), equalTo(true));
assertThat(f.isIndexed(), equalTo(false));
}
}

75
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/multifield/merge/JavaMultiFieldMergeTests.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.json.multifield.merge;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.mapper.json.JsonDocumentMapper;
import org.elasticsearch.index.mapper.json.JsonDocumentMapperParser;
import org.testng.annotations.Test;
import static org.elasticsearch.index.mapper.DocumentMapper.MergeFlags.*;
import static org.elasticsearch.util.io.Streams.*;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/**
* @author kimchy (shay.banon)
*/
@Test
public class JavaMultiFieldMergeTests {
@Test public void testMergeMultiField() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/json/multifield/merge/test-mapping1.json");
JsonDocumentMapper docMapper = (JsonDocumentMapper) new JsonDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
assertThat(docMapper.mappers().fullName("name").mapper().indexed(), equalTo(true));
assertThat(docMapper.mappers().fullName("name.indexed"), nullValue());
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/json/multifield/merge/test-data.json");
Document doc = docMapper.parse(json).doc();
Field f = doc.getField("name");
assertThat(f, notNullValue());
f = doc.getField("name.indexed");
assertThat(f, nullValue());
mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/json/multifield/merge/test-mapping2.json");
JsonDocumentMapper docMapper2 = (JsonDocumentMapper) new JsonDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
docMapper.merge(docMapper2, mergeFlags().simulate(true));
docMapper.merge(docMapper2, mergeFlags().simulate(false));
assertThat(docMapper.mappers().name("name").mapper().indexed(), equalTo(true));
assertThat(docMapper.mappers().fullName("name").mapper().indexed(), equalTo(true));
assertThat(docMapper.mappers().fullName("name.indexed").mapper(), notNullValue());
json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/json/multifield/merge/test-data.json");
doc = docMapper.parse(json).doc();
f = doc.getField("name");
assertThat(f, notNullValue());
f = doc.getField("name.indexed");
assertThat(f, notNullValue());
}
}

4
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/multifield/merge/test-data.json Normal file
View File

@ -0,0 +1,4 @@
{
_id : 1,
name : "some name"
}

7
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/multifield/merge/test-mapping1.json Normal file
View File

@ -0,0 +1,7 @@
{
person : {
properties : {
"name" : {type: "string", index : "analyzed", store : "yes"}
}
}
}

14
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/multifield/merge/test-mapping2.json Normal file
View File

@ -0,0 +1,14 @@
{
person : {
properties : {
"name" : {
type : "multi_field",
"fields" : {
"name" : {type: "string", index : "analyzed", store : "yes"},
"indexed" : {type: "string", index : "analyzed"},
"not_indexed" : {type: "string", index : "no", store : "yes"}
}
}
}
}
}

7
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/multifield/test-data.json Normal file
View File

@ -0,0 +1,7 @@
{
_id : 1,
name : "some name",
object1 : {
multi1 : 12
}
}

25
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/multifield/test-mapping.json Normal file
View File

@ -0,0 +1,25 @@
{
person : {
properties : {
"name" : {
type : "multi_field",
"fields" : {
"name" : {type: "string", index : "analyzed", store : "yes"},
"indexed" : {type: "string", index : "analyzed"},
"not_indexed" : {type: "string", index : "no", store : "yes"}
}
},
"object1" : {
properties : {
"multi1" : {
type : "multi-field",
"fields" : {
"number" : {type: "int"},
"string" : {type: "string", index : "not_analyzed"}
}
}
}
}
}
}
}