From 9ef34f82ea75a2f09262a3eac21089dba02fec29 Mon Sep 17 00:00:00 2001 From: kimchy Date: Wed, 6 Jul 2011 22:35:29 +0300 Subject: [PATCH] Mapping: non-string type field level boosting, closes #1097. --- .../index/mapper/core/ByteFieldMapper.java | 38 +++++++++-- .../index/mapper/core/DateFieldMapper.java | 43 ++++++++++-- .../index/mapper/core/DoubleFieldMapper.java | 38 +++++++++-- .../index/mapper/core/FloatFieldMapper.java | 38 +++++++++-- .../index/mapper/core/IntegerFieldMapper.java | 38 +++++++++-- .../index/mapper/core/LongFieldMapper.java | 38 +++++++++-- .../index/mapper/core/ShortFieldMapper.java | 38 +++++++++-- .../mapper/boost/CustomBoostMappingTests.java | 68 +++++++++++++++++++ 8 files changed, 308 insertions(+), 31 deletions(-) create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index eb0f60180e4..64204cf7732 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -175,8 +175,13 @@ public class ByteFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override protected boolean customBoost() { + return true; + } + @Override protected Fieldable parseCreateField(ParseContext context) throws IOException { byte value; + float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { @@ -191,7 +196,8 @@ public class ByteFieldMapper extends NumberFieldMapper { context.allEntries().addText(names.fullName(), Byte.toString(value), boost); } } else { - if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { if (nullValue == null) { return null; } @@ -199,14 +205,38 @@ public class ByteFieldMapper extends NumberFieldMapper { if (nullValueAsString != null && (context.includeInAll(includeInAll))) { context.allEntries().addText(names.fullName(), nullValueAsString, boost); } + } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + XContentParser.Token token; + String currentFieldName = null; + Byte objValue = nullValue; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else { + if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { + if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { + objValue = (byte) parser.shortValue(); + } + } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { + boost = parser.floatValue(); + } + } + } + if (objValue == null) { + // no value + return null; + } + value = objValue; } else { - value = (byte) context.parser().shortValue(); + value = (byte) parser.shortValue(); if (context.includeInAll(includeInAll)) { - context.allEntries().addText(names.fullName(), context.parser().text(), boost); + context.allEntries().addText(names.fullName(), parser.text(), boost); } } } - return new CustomByteNumericField(this, value); + CustomByteNumericField field = new CustomByteNumericField(this, value); + field.setBoost(boost); + return field; } @Override public FieldDataType fieldDataType() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index a97aa8b8be5..42bfab20b3c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -215,9 +215,14 @@ public class DateFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override protected boolean customBoost() { + return true; + } + @Override protected Fieldable parseCreateField(ParseContext context) throws IOException { String dateAsString = null; - long value = -1; + Long value = null; + float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue instanceof Number) { @@ -229,18 +234,40 @@ public class DateFieldMapper extends NumberFieldMapper { } } } else { - XContentParser.Token token = context.parser().currentToken(); + XContentParser parser = context.parser(); + XContentParser.Token token = parser.currentToken(); if (token == XContentParser.Token.VALUE_NULL) { dateAsString = nullValue; } else if (token == XContentParser.Token.VALUE_NUMBER) { - value = context.parser().longValue(); + value = parser.longValue(); + } else if (token == XContentParser.Token.START_OBJECT) { + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else { + if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { + if (token == XContentParser.Token.VALUE_NULL) { + dateAsString = nullValue; + } else if (token == XContentParser.Token.VALUE_NUMBER) { + value = parser.longValue(); + } else { + dateAsString = parser.text(); + } + } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { + boost = parser.floatValue(); + } + } + } } else { - dateAsString = context.parser().text(); + dateAsString = parser.text(); } } - if (value != -1) { - return new LongFieldMapper.CustomLongNumericField(this, value); + if (value != null) { + LongFieldMapper.CustomLongNumericField field = new LongFieldMapper.CustomLongNumericField(this, value); + field.setBoost(boost); + return field; } if (dateAsString == null) { @@ -251,7 +278,9 @@ public class DateFieldMapper extends NumberFieldMapper { } value = parseStringValue(dateAsString); - return new LongFieldMapper.CustomLongNumericField(this, value); + LongFieldMapper.CustomLongNumericField field = new LongFieldMapper.CustomLongNumericField(this, value); + field.setBoost(boost); + return field; } @Override public FieldDataType fieldDataType() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index e0eff6c135e..f7443e16e2f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -171,8 +171,13 @@ public class DoubleFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override protected boolean customBoost() { + return true; + } + @Override protected Fieldable parseCreateField(ParseContext context) throws IOException { double value; + float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { @@ -187,7 +192,8 @@ public class DoubleFieldMapper extends NumberFieldMapper { context.allEntries().addText(names.fullName(), Double.toString(value), boost); } } else { - if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { if (nullValue == null) { return null; } @@ -195,15 +201,39 @@ public class DoubleFieldMapper extends NumberFieldMapper { if (nullValueAsString != null && (context.includeInAll(includeInAll))) { context.allEntries().addText(names.fullName(), nullValueAsString, boost); } + } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + XContentParser.Token token; + String currentFieldName = null; + Double objValue = nullValue; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else { + if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { + if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { + objValue = parser.doubleValue(); + } + } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { + boost = parser.floatValue(); + } + } + } + if (objValue == null) { + // no value + return null; + } + value = objValue; } else { - value = context.parser().doubleValue(); + value = parser.doubleValue(); if (context.includeInAll(includeInAll)) { - context.allEntries().addText(names.fullName(), context.parser().text(), boost); + context.allEntries().addText(names.fullName(), parser.text(), boost); } } } - return new CustomDoubleNumericField(this, value); + CustomDoubleNumericField field = new CustomDoubleNumericField(this, value); + field.setBoost(boost); + return field; } @Override public FieldDataType fieldDataType() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index 7522f95269c..d1339e8fd52 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -170,8 +170,13 @@ public class FloatFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override protected boolean customBoost() { + return true; + } + @Override protected Fieldable parseCreateField(ParseContext context) throws IOException { float value; + float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { @@ -186,7 +191,8 @@ public class FloatFieldMapper extends NumberFieldMapper { context.allEntries().addText(names.fullName(), Float.toString(value), boost); } } else { - if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { if (nullValue == null) { return null; } @@ -194,15 +200,39 @@ public class FloatFieldMapper extends NumberFieldMapper { if (nullValueAsString != null && (context.includeInAll(includeInAll))) { context.allEntries().addText(names.fullName(), nullValueAsString, boost); } + } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + XContentParser.Token token; + String currentFieldName = null; + Float objValue = nullValue; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else { + if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { + if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { + objValue = parser.floatValue(); + } + } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { + boost = parser.floatValue(); + } + } + } + if (objValue == null) { + // no value + return null; + } + value = objValue; } else { - value = context.parser().floatValue(); + value = parser.floatValue(); if (context.includeInAll(includeInAll)) { - context.allEntries().addText(names.fullName(), context.parser().text(), boost); + context.allEntries().addText(names.fullName(), parser.text(), boost); } } } - return new CustomFloatNumericField(this, value); + CustomFloatNumericField field = new CustomFloatNumericField(this, value); + field.setBoost(boost); + return field; } @Override public FieldDataType fieldDataType() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index 289244d772c..39e4b64566f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -175,8 +175,13 @@ public class IntegerFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override protected boolean customBoost() { + return true; + } + @Override protected Fieldable parseCreateField(ParseContext context) throws IOException { int value; + float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { @@ -191,7 +196,8 @@ public class IntegerFieldMapper extends NumberFieldMapper { context.allEntries().addText(names.fullName(), Integer.toString(value), boost); } } else { - if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { if (nullValue == null) { return null; } @@ -199,15 +205,39 @@ public class IntegerFieldMapper extends NumberFieldMapper { if (nullValueAsString != null && (context.includeInAll(includeInAll))) { context.allEntries().addText(names.fullName(), nullValueAsString, boost); } + } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + XContentParser.Token token; + String currentFieldName = null; + Integer objValue = nullValue; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else { + if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { + if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { + objValue = parser.intValue(); + } + } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { + boost = parser.floatValue(); + } + } + } + if (objValue == null) { + // no value + return null; + } + value = objValue; } else { - value = context.parser().intValue(); + value = parser.intValue(); if (context.includeInAll(includeInAll)) { - context.allEntries().addText(names.fullName(), context.parser().text(), boost); + context.allEntries().addText(names.fullName(), parser.text(), boost); } } } - return new CustomIntegerNumericField(this, value); + CustomIntegerNumericField field = new CustomIntegerNumericField(this, value); + field.setBoost(boost); + return field; } @Override public FieldDataType fieldDataType() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index 31121d7d81e..403423b3057 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -175,8 +175,13 @@ public class LongFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override protected boolean customBoost() { + return true; + } + @Override protected Fieldable parseCreateField(ParseContext context) throws IOException { long value; + float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { @@ -191,7 +196,8 @@ public class LongFieldMapper extends NumberFieldMapper { context.allEntries().addText(names.fullName(), Long.toString(value), boost); } } else { - if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { if (nullValue == null) { return null; } @@ -199,14 +205,38 @@ public class LongFieldMapper extends NumberFieldMapper { if (nullValueAsString != null && (context.includeInAll(includeInAll))) { context.allEntries().addText(names.fullName(), nullValueAsString, boost); } + } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + XContentParser.Token token; + String currentFieldName = null; + Long objValue = nullValue; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else { + if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { + if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { + objValue = parser.longValue(); + } + } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { + boost = parser.floatValue(); + } + } + } + if (objValue == null) { + // no value + return null; + } + value = objValue; } else { - value = context.parser().longValue(); + value = parser.longValue(); if (context.includeInAll(includeInAll)) { - context.allEntries().addText(names.fullName(), context.parser().text(), boost); + context.allEntries().addText(names.fullName(), parser.text(), boost); } } } - return new CustomLongNumericField(this, value); + CustomLongNumericField field = new CustomLongNumericField(this, value); + field.setBoost(boost); + return field; } @Override public FieldDataType fieldDataType() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 44b6350a1bf..465d8824ee6 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -175,8 +175,13 @@ public class ShortFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override protected boolean customBoost() { + return true; + } + @Override protected Fieldable parseCreateField(ParseContext context) throws IOException { short value; + float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { @@ -191,7 +196,8 @@ public class ShortFieldMapper extends NumberFieldMapper { context.allEntries().addText(names.fullName(), Short.toString(value), boost); } } else { - if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { if (nullValue == null) { return null; } @@ -199,14 +205,38 @@ public class ShortFieldMapper extends NumberFieldMapper { if (nullValueAsString != null && (context.includeInAll(includeInAll))) { context.allEntries().addText(names.fullName(), nullValueAsString, boost); } + } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + XContentParser.Token token; + String currentFieldName = null; + Short objValue = nullValue; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else { + if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { + if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { + objValue = parser.shortValue(); + } + } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { + boost = parser.floatValue(); + } + } + } + if (objValue == null) { + // no value + return null; + } + value = objValue; } else { - value = context.parser().shortValue(); + value = parser.shortValue(); if (context.includeInAll(includeInAll)) { - context.allEntries().addText(names.fullName(), context.parser().text(), boost); + context.allEntries().addText(names.fullName(), parser.text(), boost); } } } - return new CustomShortNumericField(this, value); + CustomShortNumericField field = new CustomShortNumericField(this, value); + field.setBoost(boost); + return field; } @Override public FieldDataType fieldDataType() { diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java new file mode 100644 index 00000000000..4bf12ea0ca1 --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.boost; + +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperTests; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.testng.annotations.Test; + +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +@Test +public class CustomBoostMappingTests { + + @Test public void testCustomBoostValues() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("s_field").field("type", "string").endObject() + .startObject("l_field").field("type", "long").endObject() + .startObject("i_field").field("type", "integer").endObject() + .startObject("sh_field").field("type", "short").endObject() + .startObject("b_field").field("type", "byte").endObject() + .startObject("d_field").field("type", "double").endObject() + .startObject("f_field").field("type", "float").endObject() + .startObject("date_field").field("type", "date").endObject() + .endObject().endObject().endObject().string(); + + DocumentMapper mapper = MapperTests.newParser().parse(mapping); + + ParsedDocument doc = mapper.parse("type", "1", XContentFactory.jsonBuilder().startObject() + .startObject("s_field").field("value", "s_value").field("boost", 2.0f).endObject() + .startObject("l_field").field("value", 1l).field("boost", 3.0f).endObject() + .startObject("i_field").field("value", 1).field("boost", 4.0f).endObject() + .startObject("sh_field").field("value", 1).field("boost", 5.0f).endObject() + .startObject("b_field").field("value", 1).field("boost", 6.0f).endObject() + .startObject("d_field").field("value", 1).field("boost", 7.0f).endObject() + .startObject("f_field").field("value", 1).field("boost", 8.0f).endObject() + .startObject("date_field").field("value", "20100101").field("boost", 9.0f).endObject() + .endObject().copiedBytes()); + + assertThat(doc.masterDoc().getFieldable("s_field").getBoost(), equalTo(2.0f)); + assertThat(doc.masterDoc().getFieldable("l_field").getBoost(), equalTo(3.0f)); + assertThat(doc.masterDoc().getFieldable("i_field").getBoost(), equalTo(4.0f)); + assertThat(doc.masterDoc().getFieldable("sh_field").getBoost(), equalTo(5.0f)); + assertThat(doc.masterDoc().getFieldable("b_field").getBoost(), equalTo(6.0f)); + assertThat(doc.masterDoc().getFieldable("d_field").getBoost(), equalTo(7.0f)); + assertThat(doc.masterDoc().getFieldable("f_field").getBoost(), equalTo(8.0f)); + assertThat(doc.masterDoc().getFieldable("date_field").getBoost(), equalTo(9.0f)); + } +} \ No newline at end of file