honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Mapping: Add _size field mapping, indexing the original source size, closes #804.

This commit is contained in:
kimchy 2011-03-23 17:37:40 +02:00
parent b2d7cd78b7
commit 20593fb966
5 changed files with 248 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View File

@ -46,6 +46,10 @@ public interface FieldMapper<T> {
private final String fullName;
public Names(String name) {
this(name, name, name, name);
}
public Names(String name, String indexName, String indexNameClean, String fullName) {
this.name = name.intern();
this.indexName = indexName.intern();

109
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/SizeFieldMapper.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.xcontent;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.MergeMappingException;
import java.io.IOException;
public class SizeFieldMapper extends IntegerFieldMapper {
public static final String CONTENT_TYPE = "_size";
public static class Defaults extends IntegerFieldMapper.Defaults {
public static final String NAME = CONTENT_TYPE;
public static final boolean ENABLED = false;
}
public static class Builder extends XContentMapper.Builder<Builder, IntegerFieldMapper> {
protected boolean enabled = Defaults.ENABLED;
protected Field.Store store = Defaults.STORE;
public Builder() {
super(Defaults.NAME);
builder = this;
}
public Builder enabled(boolean enabled) {
this.enabled = enabled;
return builder;
}
public Builder store(Field.Store store) {
this.store = store;
return builder;
}
@Override public SizeFieldMapper build(BuilderContext context) {
return new SizeFieldMapper(enabled, store);
}
}
private final boolean enabled;
public SizeFieldMapper() {
this(Defaults.ENABLED, Defaults.STORE);
}
public SizeFieldMapper(boolean enabled, Field.Store store) {
super(new Names(Defaults.NAME), Defaults.PRECISION_STEP, Defaults.INDEX, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS, Defaults.NULL_VALUE);
this.enabled = enabled;
}
@Override protected String contentType() {
return Defaults.NAME;
}
public boolean enabled() {
return this.enabled;
}
@Override protected Fieldable parseCreateField(ParseContext context) throws IOException {
if (!enabled) {
return null;
}
return new CustomIntegerNumericField(this, ((Number) context.externalValue()).intValue());
}
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
// all are defaults, no need to write it at all
if (enabled == Defaults.ENABLED && store == Defaults.STORE) {
return builder;
}
builder.startObject(contentType());
if (enabled != Defaults.ENABLED) {
builder.field("enabled", enabled);
}
if (store != Defaults.STORE) {
builder.field("store", store.name().toLowerCase());
}
builder.endObject();
return builder;
}
@Override public void merge(XContentMapper mergeWith, MergeContext mergeContext) throws MergeMappingException {
// maybe allow to change enabled? But then we need to figure out null for default value
}
}

21
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java
View File

@ -57,6 +57,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
private IndexFieldMapper indexFieldMapper = new IndexFieldMapper();
private SourceFieldMapper sourceFieldMapper = new SourceFieldMapper();
private SizeFieldMapper sizeFieldMapper = new SizeFieldMapper();
private RoutingFieldMapper routingFieldMapper = new RoutingFieldMapper();
@ -95,6 +96,11 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
return this;
}
public Builder sizeField(SizeFieldMapper.Builder builder) {
this.sizeFieldMapper = builder.build(builderContext);
return this;
}
public Builder idField(IdFieldMapper.Builder builder) {
this.idFieldMapper = builder.build(builderContext);
return this;
@ -161,7 +167,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
public XContentDocumentMapper build(XContentDocumentMapperParser docMapperParser) {
Preconditions.checkNotNull(rootObjectMapper, "Mapper builder must have the root object mapper set");
return new XContentDocumentMapper(index, docMapperParser, rootObjectMapper, meta, uidFieldMapper, idFieldMapper, typeFieldMapper, indexFieldMapper,
sourceFieldMapper, parentFieldMapper, routingFieldMapper, allFieldMapper, analyzerMapper, indexAnalyzer, searchAnalyzer, boostFieldMapper);
sourceFieldMapper, sizeFieldMapper, parentFieldMapper, routingFieldMapper, allFieldMapper, analyzerMapper, indexAnalyzer, searchAnalyzer, boostFieldMapper);
}
}
@ -191,6 +197,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
private final IndexFieldMapper indexFieldMapper;
private final SourceFieldMapper sourceFieldMapper;
private final SizeFieldMapper sizeFieldMapper;
private final RoutingFieldMapper routingFieldMapper;
@ -224,6 +231,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
TypeFieldMapper typeFieldMapper,
IndexFieldMapper indexFieldMapper,
SourceFieldMapper sourceFieldMapper,
SizeFieldMapper sizeFieldMapper,
@Nullable ParentFieldMapper parentFieldMapper,
RoutingFieldMapper routingFieldMapper,
AllFieldMapper allFieldMapper,
@ -240,6 +248,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
this.typeFieldMapper = typeFieldMapper;
this.indexFieldMapper = indexFieldMapper;
this.sourceFieldMapper = sourceFieldMapper;
this.sizeFieldMapper = sizeFieldMapper;
this.parentFieldMapper = parentFieldMapper;
this.routingFieldMapper = routingFieldMapper;
this.allFieldMapper = allFieldMapper;
@ -269,6 +278,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
}
tempFieldMappers.add(typeFieldMapper);
tempFieldMappers.add(sourceFieldMapper);
tempFieldMappers.add(sizeFieldMapper);
tempFieldMappers.add(uidFieldMapper);
tempFieldMappers.add(allFieldMapper);
// now traverse and get all the statically defined ones
@ -410,6 +420,11 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
// }
}
if (sizeFieldMapper.enabled()) {
context.externalValue(source.source().length);
sizeFieldMapper.parse(context);
}
if (sourceFieldMapper.enabled()) {
sourceFieldMapper.parse(context);
}
@ -493,6 +508,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
fieldMapperListener.fieldMapper(indexFieldMapper);
}
fieldMapperListener.fieldMapper(sourceFieldMapper);
fieldMapperListener.fieldMapper(sizeFieldMapper);
fieldMapperListener.fieldMapper(typeFieldMapper);
fieldMapperListener.fieldMapper(uidFieldMapper);
fieldMapperListener.fieldMapper(allFieldMapper);
@ -509,6 +525,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
allFieldMapper.merge(xContentMergeWith.allFieldMapper, mergeContext);
analyzerMapper.merge(xContentMergeWith.analyzerMapper, mergeContext);
sourceFieldMapper.merge(xContentMergeWith.sourceFieldMapper, mergeContext);
sizeFieldMapper.merge(xContentMergeWith.sizeFieldMapper, mergeContext);
if (!mergeFlags.simulate()) {
// let the merge with attributes to override the attributes
@ -559,7 +576,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
}
// no need to pass here id and boost, since they are added to the root object mapper
// in the constructor
}, indexFieldMapper, typeFieldMapper, allFieldMapper, analyzerMapper, sourceFieldMapper);
}, indexFieldMapper, typeFieldMapper, allFieldMapper, analyzerMapper, sourceFieldMapper, sizeFieldMapper);
return builder;
}
}

17
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java
View File

@ -140,6 +140,8 @@ public class XContentDocumentMapperParser extends AbstractIndexComponent impleme
if (SourceFieldMapper.CONTENT_TYPE.equals(fieldName) || "sourceField".equals(fieldName)) {
docBuilder.sourceField(parseSourceField((Map<String, Object>) fieldNode, parserContext));
} else if (SizeFieldMapper.CONTENT_TYPE.equals(fieldName)) {
docBuilder.sizeField(parseSizeField((Map<String, Object>) fieldNode, parserContext));
} else if (IdFieldMapper.CONTENT_TYPE.equals(fieldName) || "idField".equals(fieldName)) {
docBuilder.idField(parseIdField((Map<String, Object>) fieldNode, parserContext));
} else if (IndexFieldMapper.CONTENT_TYPE.equals(fieldName) || "indexField".equals(fieldName)) {
@ -271,6 +273,21 @@ public class XContentDocumentMapperParser extends AbstractIndexComponent impleme
return builder;
}
private SizeFieldMapper.Builder parseSizeField(Map<String, Object> node, XContentMapper.TypeParser.ParserContext parserContext) {
SizeFieldMapper.Builder builder = new SizeFieldMapper.Builder();
for (Map.Entry<String, Object> entry : node.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("store")) {
builder.store(parseStore(fieldName, fieldNode.toString()));
}
}
return builder;
}
private SourceFieldMapper.Builder parseSourceField(Map<String, Object> sourceNode, XContentMapper.TypeParser.ParserContext parserContext) {
SourceFieldMapper.Builder builder = source();

99
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/size/SizeMappingTests.java Normal file
View File

@ -0,0 +1,99 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.xcontent.size;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.xcontent.MapperTests;
import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
import org.testng.annotations.Test;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
@Test
public class SizeMappingTests {
@Test public void testSizeEnabled() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_size").field("enabled", true).endObject()
.endObject().endObject().string();
XContentDocumentMapper docMapper = MapperTests.newParser().parse(mapping);
byte[] source = XContentFactory.jsonBuilder()
.startObject()
.field("field", "value")
.endObject()
.copiedBytes();
ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
assertThat(doc.doc().getFieldable("_size").isStored(), equalTo(false));
assertThat(doc.doc().getFieldable("_size").tokenStreamValue(), notNullValue());
}
@Test public void testSizeEnabledAndStored() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_size").field("enabled", true).field("store", "yes").endObject()
.endObject().endObject().string();
XContentDocumentMapper docMapper = MapperTests.newParser().parse(mapping);
byte[] source = XContentFactory.jsonBuilder()
.startObject()
.field("field", "value")
.endObject()
.copiedBytes();
ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
assertThat(doc.doc().getFieldable("_size").isStored(), equalTo(true));
assertThat(doc.doc().getFieldable("_size").tokenStreamValue(), notNullValue());
}
@Test public void testSizeDisabled() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_size").field("enabled", false).endObject()
.endObject().endObject().string();
XContentDocumentMapper docMapper = MapperTests.newParser().parse(mapping);
byte[] source = XContentFactory.jsonBuilder()
.startObject()
.field("field", "value")
.endObject()
.copiedBytes();
ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
assertThat(doc.doc().getFieldable("_size"), nullValue());
}
@Test public void testSizeNotSet() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.endObject().endObject().string();
XContentDocumentMapper docMapper = MapperTests.newParser().parse(mapping);
byte[] source = XContentFactory.jsonBuilder()
.startObject()
.field("field", "value")
.endObject()
.copiedBytes();
ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
assertThat(doc.doc().getFieldable("_size"), nullValue());
}
}