From 20593fb9663029e37d65bc6b83f98d6c3778a088 Mon Sep 17 00:00:00 2001 From: kimchy Date: Wed, 23 Mar 2011 17:37:40 +0200 Subject: [PATCH] Mapping: Add _size field mapping, indexing the original source size, closes #804. --- .../index/mapper/FieldMapper.java | 4 + .../mapper/xcontent/SizeFieldMapper.java | 109 ++++++++++++++++++ .../xcontent/XContentDocumentMapper.java | 21 +++- .../XContentDocumentMapperParser.java | 17 +++ .../xcontent/size/SizeMappingTests.java | 99 ++++++++++++++++ 5 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/SizeFieldMapper.java create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/size/SizeMappingTests.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 75d4b7dab16..b7cc0e8b127 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -46,6 +46,10 @@ public interface FieldMapper { private final String fullName; + public Names(String name) { + this(name, name, name, name); + } + public Names(String name, String indexName, String indexNameClean, String fullName) { this.name = name.intern(); this.indexName = indexName.intern(); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/SizeFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/SizeFieldMapper.java new file mode 100644 index 00000000000..9696ddef87e --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/SizeFieldMapper.java @@ -0,0 +1,109 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.xcontent; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Fieldable; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.mapper.MergeMappingException; + +import java.io.IOException; + +public class SizeFieldMapper extends IntegerFieldMapper { + + public static final String CONTENT_TYPE = "_size"; + + public static class Defaults extends IntegerFieldMapper.Defaults { + public static final String NAME = CONTENT_TYPE; + public static final boolean ENABLED = false; + } + + public static class Builder extends XContentMapper.Builder { + + protected boolean enabled = Defaults.ENABLED; + + protected Field.Store store = Defaults.STORE; + + public Builder() { + super(Defaults.NAME); + builder = this; + } + + public Builder enabled(boolean enabled) { + this.enabled = enabled; + return builder; + } + + public Builder store(Field.Store store) { + this.store = store; + return builder; + } + + @Override public SizeFieldMapper build(BuilderContext context) { + return new SizeFieldMapper(enabled, store); + } + } + + private final boolean enabled; + + public SizeFieldMapper() { + this(Defaults.ENABLED, Defaults.STORE); + } + + public SizeFieldMapper(boolean enabled, Field.Store store) { + super(new Names(Defaults.NAME), Defaults.PRECISION_STEP, Defaults.INDEX, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS, Defaults.NULL_VALUE); + this.enabled = enabled; + } + + @Override protected String contentType() { + return Defaults.NAME; + } + + public boolean enabled() { + return this.enabled; + } + + @Override protected Fieldable parseCreateField(ParseContext context) throws IOException { + if (!enabled) { + return null; + } + return new CustomIntegerNumericField(this, ((Number) context.externalValue()).intValue()); + } + + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // all are defaults, no need to write it at all + if (enabled == Defaults.ENABLED && store == Defaults.STORE) { + return builder; + } + builder.startObject(contentType()); + if (enabled != Defaults.ENABLED) { + builder.field("enabled", enabled); + } + if (store != Defaults.STORE) { + builder.field("store", store.name().toLowerCase()); + } + builder.endObject(); + return builder; + } + + @Override public void merge(XContentMapper mergeWith, MergeContext mergeContext) throws MergeMappingException { + // maybe allow to change enabled? But then we need to figure out null for default value + } +} \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java index 46fa78b38a0..64a408085ba 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java @@ -57,6 +57,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { private IndexFieldMapper indexFieldMapper = new IndexFieldMapper(); private SourceFieldMapper sourceFieldMapper = new SourceFieldMapper(); + private SizeFieldMapper sizeFieldMapper = new SizeFieldMapper(); private RoutingFieldMapper routingFieldMapper = new RoutingFieldMapper(); @@ -95,6 +96,11 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { return this; } + public Builder sizeField(SizeFieldMapper.Builder builder) { + this.sizeFieldMapper = builder.build(builderContext); + return this; + } + public Builder idField(IdFieldMapper.Builder builder) { this.idFieldMapper = builder.build(builderContext); return this; @@ -161,7 +167,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { public XContentDocumentMapper build(XContentDocumentMapperParser docMapperParser) { Preconditions.checkNotNull(rootObjectMapper, "Mapper builder must have the root object mapper set"); return new XContentDocumentMapper(index, docMapperParser, rootObjectMapper, meta, uidFieldMapper, idFieldMapper, typeFieldMapper, indexFieldMapper, - sourceFieldMapper, parentFieldMapper, routingFieldMapper, allFieldMapper, analyzerMapper, indexAnalyzer, searchAnalyzer, boostFieldMapper); + sourceFieldMapper, sizeFieldMapper, parentFieldMapper, routingFieldMapper, allFieldMapper, analyzerMapper, indexAnalyzer, searchAnalyzer, boostFieldMapper); } } @@ -191,6 +197,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { private final IndexFieldMapper indexFieldMapper; private final SourceFieldMapper sourceFieldMapper; + private final SizeFieldMapper sizeFieldMapper; private final RoutingFieldMapper routingFieldMapper; @@ -224,6 +231,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { TypeFieldMapper typeFieldMapper, IndexFieldMapper indexFieldMapper, SourceFieldMapper sourceFieldMapper, + SizeFieldMapper sizeFieldMapper, @Nullable ParentFieldMapper parentFieldMapper, RoutingFieldMapper routingFieldMapper, AllFieldMapper allFieldMapper, @@ -240,6 +248,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { this.typeFieldMapper = typeFieldMapper; this.indexFieldMapper = indexFieldMapper; this.sourceFieldMapper = sourceFieldMapper; + this.sizeFieldMapper = sizeFieldMapper; this.parentFieldMapper = parentFieldMapper; this.routingFieldMapper = routingFieldMapper; this.allFieldMapper = allFieldMapper; @@ -269,6 +278,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { } tempFieldMappers.add(typeFieldMapper); tempFieldMappers.add(sourceFieldMapper); + tempFieldMappers.add(sizeFieldMapper); tempFieldMappers.add(uidFieldMapper); tempFieldMappers.add(allFieldMapper); // now traverse and get all the statically defined ones @@ -410,6 +420,11 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { // } } + if (sizeFieldMapper.enabled()) { + context.externalValue(source.source().length); + sizeFieldMapper.parse(context); + } + if (sourceFieldMapper.enabled()) { sourceFieldMapper.parse(context); } @@ -493,6 +508,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { fieldMapperListener.fieldMapper(indexFieldMapper); } fieldMapperListener.fieldMapper(sourceFieldMapper); + fieldMapperListener.fieldMapper(sizeFieldMapper); fieldMapperListener.fieldMapper(typeFieldMapper); fieldMapperListener.fieldMapper(uidFieldMapper); fieldMapperListener.fieldMapper(allFieldMapper); @@ -509,6 +525,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { allFieldMapper.merge(xContentMergeWith.allFieldMapper, mergeContext); analyzerMapper.merge(xContentMergeWith.analyzerMapper, mergeContext); sourceFieldMapper.merge(xContentMergeWith.sourceFieldMapper, mergeContext); + sizeFieldMapper.merge(xContentMergeWith.sizeFieldMapper, mergeContext); if (!mergeFlags.simulate()) { // let the merge with attributes to override the attributes @@ -559,7 +576,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { } // no need to pass here id and boost, since they are added to the root object mapper // in the constructor - }, indexFieldMapper, typeFieldMapper, allFieldMapper, analyzerMapper, sourceFieldMapper); + }, indexFieldMapper, typeFieldMapper, allFieldMapper, analyzerMapper, sourceFieldMapper, sizeFieldMapper); return builder; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java index 6c7b1f6de83..d4dea6df121 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java @@ -140,6 +140,8 @@ public class XContentDocumentMapperParser extends AbstractIndexComponent impleme if (SourceFieldMapper.CONTENT_TYPE.equals(fieldName) || "sourceField".equals(fieldName)) { docBuilder.sourceField(parseSourceField((Map) fieldNode, parserContext)); + } else if (SizeFieldMapper.CONTENT_TYPE.equals(fieldName)) { + docBuilder.sizeField(parseSizeField((Map) fieldNode, parserContext)); } else if (IdFieldMapper.CONTENT_TYPE.equals(fieldName) || "idField".equals(fieldName)) { docBuilder.idField(parseIdField((Map) fieldNode, parserContext)); } else if (IndexFieldMapper.CONTENT_TYPE.equals(fieldName) || "indexField".equals(fieldName)) { @@ -271,6 +273,21 @@ public class XContentDocumentMapperParser extends AbstractIndexComponent impleme return builder; } + private SizeFieldMapper.Builder parseSizeField(Map node, XContentMapper.TypeParser.ParserContext parserContext) { + SizeFieldMapper.Builder builder = new SizeFieldMapper.Builder(); + + for (Map.Entry entry : node.entrySet()) { + String fieldName = Strings.toUnderscoreCase(entry.getKey()); + Object fieldNode = entry.getValue(); + if (fieldName.equals("enabled")) { + builder.enabled(nodeBooleanValue(fieldNode)); + } else if (fieldName.equals("store")) { + builder.store(parseStore(fieldName, fieldNode.toString())); + } + } + return builder; + } + private SourceFieldMapper.Builder parseSourceField(Map sourceNode, XContentMapper.TypeParser.ParserContext parserContext) { SourceFieldMapper.Builder builder = source(); diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/size/SizeMappingTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/size/SizeMappingTests.java new file mode 100644 index 00000000000..8795c9a4678 --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/size/SizeMappingTests.java @@ -0,0 +1,99 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.xcontent.size; + +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.xcontent.MapperTests; +import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper; +import org.testng.annotations.Test; + +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +@Test +public class SizeMappingTests { + + @Test public void testSizeEnabled() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_size").field("enabled", true).endObject() + .endObject().endObject().string(); + XContentDocumentMapper docMapper = MapperTests.newParser().parse(mapping); + + byte[] source = XContentFactory.jsonBuilder() + .startObject() + .field("field", "value") + .endObject() + .copiedBytes(); + ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1")); + + assertThat(doc.doc().getFieldable("_size").isStored(), equalTo(false)); + assertThat(doc.doc().getFieldable("_size").tokenStreamValue(), notNullValue()); + } + + @Test public void testSizeEnabledAndStored() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_size").field("enabled", true).field("store", "yes").endObject() + .endObject().endObject().string(); + XContentDocumentMapper docMapper = MapperTests.newParser().parse(mapping); + + byte[] source = XContentFactory.jsonBuilder() + .startObject() + .field("field", "value") + .endObject() + .copiedBytes(); + ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1")); + + assertThat(doc.doc().getFieldable("_size").isStored(), equalTo(true)); + assertThat(doc.doc().getFieldable("_size").tokenStreamValue(), notNullValue()); + } + + @Test public void testSizeDisabled() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_size").field("enabled", false).endObject() + .endObject().endObject().string(); + XContentDocumentMapper docMapper = MapperTests.newParser().parse(mapping); + + byte[] source = XContentFactory.jsonBuilder() + .startObject() + .field("field", "value") + .endObject() + .copiedBytes(); + ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1")); + + assertThat(doc.doc().getFieldable("_size"), nullValue()); + } + + @Test public void testSizeNotSet() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .endObject().endObject().string(); + XContentDocumentMapper docMapper = MapperTests.newParser().parse(mapping); + + byte[] source = XContentFactory.jsonBuilder() + .startObject() + .field("field", "value") + .endObject() + .copiedBytes(); + ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1")); + + assertThat(doc.doc().getFieldable("_size"), nullValue()); + } +} \ No newline at end of file