From 1dd59978896e1c5637e492563100d08dc99211db Mon Sep 17 00:00:00 2001 From: kimchy Date: Tue, 16 Mar 2010 23:04:20 +0200 Subject: [PATCH] _all field, closes #63. --- .../action/terms/TermsRequest.java | 25 +- .../index/mapper/AllFieldMapper.java | 30 +++ .../index/mapper/DocumentMapper.java | 2 + .../index/mapper/json/JsonAllFieldMapper.java | 154 ++++++++++++ .../mapper/json/JsonDateFieldMapper.java | 12 +- .../index/mapper/json/JsonDocumentMapper.java | 26 +- .../mapper/json/JsonDocumentMapperParser.java | 21 ++ .../mapper/json/JsonDoubleFieldMapper.java | 18 +- .../index/mapper/json/JsonFieldMapper.java | 7 + .../mapper/json/JsonFloatFieldMapper.java | 16 +- .../index/mapper/json/JsonIdFieldMapper.java | 9 +- .../mapper/json/JsonIncludeInAllMapper.java | 28 +++ .../mapper/json/JsonIntegerFieldMapper.java | 18 +- .../mapper/json/JsonLongFieldMapper.java | 18 +- .../index/mapper/json/JsonMapperBuilders.java | 4 + .../mapper/json/JsonMultiFieldMapper.java | 19 +- .../mapper/json/JsonNumberFieldMapper.java | 20 +- .../index/mapper/json/JsonObjectMapper.java | 34 ++- .../index/mapper/json/JsonParseContext.java | 8 + .../mapper/json/JsonStringFieldMapper.java | 25 +- .../index/query/json/JsonQueryBuilders.java | 8 + .../json/MoreLikeThisJsonQueryBuilder.java | 17 +- .../json/MoreLikeThisJsonQueryParser.java | 2 + .../json/QueryStringJsonQueryParser.java | 5 +- .../util/lucene/all/AllEntries.java | 159 ++++++++++++ .../util/lucene/all/AllTermQuery.java | 188 ++++++++++++++ .../util/lucene/all/AllTokenFilter.java | 67 +++++ .../mapper/json/all/SimpleAllMapperTests.java | 70 ++++++ .../index/mapper/json/all/mapping.json | 33 +++ .../index/mapper/json/all/test1.json | 20 ++ .../util/lucene/all/SimpleAllTests.java | 231 ++++++++++++++++++ .../integration/terms/TermsActionTests.java | 11 + 32 files changed, 1256 insertions(+), 49 deletions(-) create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonAllFieldMapper.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIncludeInAllMapper.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllEntries.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTermQuery.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenFilter.java create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/SimpleAllMapperTests.java create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/mapping.json create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/test1.json create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/util/lucene/all/SimpleAllTests.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/terms/TermsRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/terms/TermsRequest.java index e0606220f70..99eff672624 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/terms/TermsRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/terms/TermsRequest.java @@ -20,20 +20,19 @@ package org.elasticsearch.action.terms; import org.elasticsearch.ElasticSearchIllegalArgumentException; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.broadcast.BroadcastOperationRequest; -import org.elasticsearch.util.Required; +import org.elasticsearch.index.mapper.AllFieldMapper; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import static org.elasticsearch.action.Actions.*; - /** * Terms request represent a request to get terms in one or more indices of specific fields and their * document frequencies (in how many document each term exists). * + *

By default, the "_all" field will be used to extract terms and frequencies. + * *

This is very handy to implement things like tag clouds and auto complete (using {@link #prefix(String)} or * {@link #regexp(String)}). * @@ -103,7 +102,9 @@ public class TermsRequest extends BroadcastOperationRequest { } } - private String[] fields; + private static final String[] DEFAULT_FIELDS = new String[]{AllFieldMapper.NAME}; + + private String[] fields = DEFAULT_FIELDS; private String from; @@ -134,20 +135,12 @@ public class TermsRequest extends BroadcastOperationRequest { /** * Constructs a new terms requests with the provided indices. Don't pass anything for it to run - * over all the indices. Note, the {@link #fields(String...)} is required. + * over all the indices. */ public TermsRequest(String... indices) { super(indices, null); } - @Override public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = super.validate(); - if (fields == null || fields.length == 0) { - validationException = addValidationError("fields is missing", validationException); - } - return validationException; - } - /** * The fields within each document which terms will be iterated over and returned with the * document frequencies. @@ -158,9 +151,9 @@ public class TermsRequest extends BroadcastOperationRequest { /** * The fields within each document which terms will be iterated over and returned with the - * document frequencies. + * document frequencies. By default will use the "_all" field. */ - @Required public TermsRequest fields(String... fields) { + public TermsRequest fields(String... fields) { this.fields = fields; return this; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java new file mode 100644 index 00000000000..cc76e5a8940 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java @@ -0,0 +1,30 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.util.StringHelper; + +/** + * @author kimchy (shay.banon) + */ +public interface AllFieldMapper extends FieldMapper, InternalMapper { + + public static final String NAME = StringHelper.intern("_all"); +} \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 47546aa345b..db5dc9e3731 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -53,6 +53,8 @@ public interface DocumentMapper { BoostFieldMapper boostMapper(); + AllFieldMapper allFieldMapper(); + DocumentFieldMappers mappers(); /** diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonAllFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonAllFieldMapper.java new file mode 100644 index 00000000000..bddaf46701e --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonAllFieldMapper.java @@ -0,0 +1,154 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.json; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Fieldable; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.AllFieldMapper; +import org.elasticsearch.index.mapper.MergeMappingException; +import org.elasticsearch.util.json.JsonBuilder; +import org.elasticsearch.util.lucene.Lucene; + +import java.io.IOException; + +import static org.elasticsearch.util.lucene.all.AllTokenFilter.*; + +/** + * @author kimchy (shay.banon) + */ +public class JsonAllFieldMapper extends JsonFieldMapper implements AllFieldMapper { + + public static final String JSON_TYPE = "allField"; + + public static class Defaults extends JsonFieldMapper.Defaults { + public static final String NAME = AllFieldMapper.NAME; + public static final String INDEX_NAME = AllFieldMapper.NAME; + public static final boolean ENABLED = true; + } + + + public static class Builder extends JsonFieldMapper.Builder { + + private boolean enabled = Defaults.ENABLED; + + public Builder() { + super(Defaults.NAME); + builder = this; + indexName = Defaults.INDEX_NAME; + } + + public Builder enabled(boolean enabled) { + this.enabled = enabled; + return this; + } + + @Override public Builder termVector(Field.TermVector termVector) { + return super.termVector(termVector); + } + + @Override protected Builder indexAnalyzer(NamedAnalyzer indexAnalyzer) { + return super.indexAnalyzer(indexAnalyzer); + } + + @Override protected Builder searchAnalyzer(NamedAnalyzer searchAnalyzer) { + return super.searchAnalyzer(searchAnalyzer); + } + + @Override public JsonAllFieldMapper build(BuilderContext context) { + return new JsonAllFieldMapper(name, termVector, omitNorms, omitTermFreqAndPositions, + indexAnalyzer, searchAnalyzer, enabled); + } + } + + + private boolean enabled; + + public JsonAllFieldMapper() { + this(Defaults.NAME, Defaults.TERM_VECTOR, Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS, null, null, Defaults.ENABLED); + } + + protected JsonAllFieldMapper(String name, Field.TermVector termVector, boolean omitNorms, boolean omitTermFreqAndPositions, + NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, boolean enabled) { + super(new Names(name, name, name, name), Field.Index.ANALYZED, Field.Store.NO, termVector, 1.0f, omitNorms, omitTermFreqAndPositions, + indexAnalyzer, searchAnalyzer); + this.enabled = enabled; + } + + public boolean enabled() { + return this.enabled; + } + + @Override protected Field parseCreateField(JsonParseContext jsonContext) throws IOException { + if (!enabled) { + return null; + } + Analyzer analyzer = indexAnalyzer(); + if (analyzer == null) { + analyzer = jsonContext.docMapper().indexAnalyzer(); + if (analyzer == null) { + analyzer = Lucene.STANDARD_ANALYZER; + } + } + return new Field(names.indexName(), allTokenStream(names.indexName(), jsonContext.allEntries().finishTexts(), analyzer), termVector); + } + + @Override public Void value(Fieldable field) { + return null; + } + + @Override public String valueAsString(Fieldable field) { + return null; + } + + @Override public Object valueForSearch(Fieldable field) { + return null; + } + + @Override public String indexedValue(String value) { + return null; + } + + @Override public String indexedValue(Void value) { + return null; + } + + @Override protected String jsonType() { + return JSON_TYPE; + } + + @Override public void toJson(JsonBuilder builder, Params params) throws IOException { + builder.startObject(JSON_TYPE); + builder.field("enabled", enabled); + builder.field("termVector", termVector.name().toLowerCase()); + if (indexAnalyzer != null && !indexAnalyzer.name().startsWith("_")) { + builder.field("indexAnalyzer", indexAnalyzer.name()); + } + if (searchAnalyzer != null && !searchAnalyzer.name().startsWith("_")) { + builder.field("searchAnalyzer", searchAnalyzer.name()); + } + builder.endObject(); + } + + @Override public void merge(JsonMapper mergeWith, JsonMergeContext mergeContext) throws MergeMappingException { + // do nothing here, no merging, but also no exception + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDateFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDateFieldMapper.java index e149a5bf9b6..0d2490285b6 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDateFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDateFieldMapper.java @@ -34,7 +34,7 @@ import org.elasticsearch.util.json.JsonBuilder; import java.io.IOException; /** - * @author kimchy (Shay Banon) + * @author kimchy (shay.banon) */ public class JsonDateFieldMapper extends JsonNumberFieldMapper { @@ -68,8 +68,10 @@ public class JsonDateFieldMapper extends JsonNumberFieldMapper { } @Override public JsonDateFieldMapper build(BuilderContext context) { - return new JsonDateFieldMapper(buildNames(context), dateTimeFormatter, + JsonDateFieldMapper fieldMapper = new JsonDateFieldMapper(buildNames(context), dateTimeFormatter, precisionStep, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue); + fieldMapper.includeInAll(includeInAll); + return fieldMapper; } } @@ -137,6 +139,9 @@ public class JsonDateFieldMapper extends JsonNumberFieldMapper { if (dateAsString == null) { return null; } + if (includeInAll == null || includeInAll) { + jsonContext.allEntries().addText(names.fullName(), dateAsString, boost); + } long value = dateTimeFormatter.parser().parseMillis(dateAsString); Field field = null; if (stored()) { @@ -164,5 +169,8 @@ public class JsonDateFieldMapper extends JsonNumberFieldMapper { if (nullValue != null) { builder.field("nullValue", nullValue); } + if (includeInAll != null) { + builder.field("includeInAll", includeInAll); + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDocumentMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDocumentMapper.java index 68865d40244..afff2beeb28 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDocumentMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDocumentMapper.java @@ -40,7 +40,7 @@ import static com.google.common.collect.Lists.*; import static org.elasticsearch.util.json.JsonBuilder.*; /** - * @author kimchy (Shay Banon) + * @author kimchy (shay.banon) */ public class JsonDocumentMapper implements DocumentMapper, ToJson { @@ -56,6 +56,8 @@ public class JsonDocumentMapper implements DocumentMapper, ToJson { private JsonBoostFieldMapper boostFieldMapper = new JsonBoostFieldMapper(); + private JsonAllFieldMapper allFieldMapper = new JsonAllFieldMapper(); + private Analyzer indexAnalyzer; private Analyzer searchAnalyzer; @@ -95,6 +97,11 @@ public class JsonDocumentMapper implements DocumentMapper, ToJson { return this; } + public Builder allField(JsonAllFieldMapper.Builder builder) { + this.allFieldMapper = builder.build(builderContext); + return this; + } + public Builder mappingSource(String mappingSource) { this.mappingSource = mappingSource; return this; @@ -121,7 +128,7 @@ public class JsonDocumentMapper implements DocumentMapper, ToJson { public JsonDocumentMapper build() { Preconditions.checkNotNull(rootObjectMapper, "Json mapper builder must have the root object mapper set"); return new JsonDocumentMapper(rootObjectMapper, uidFieldMapper, idFieldMapper, typeFieldMapper, - sourceFieldMapper, indexAnalyzer, searchAnalyzer, boostFieldMapper, mappingSource); + sourceFieldMapper, allFieldMapper, indexAnalyzer, searchAnalyzer, boostFieldMapper, mappingSource); } } @@ -148,6 +155,8 @@ public class JsonDocumentMapper implements DocumentMapper, ToJson { private final JsonBoostFieldMapper boostFieldMapper; + private final JsonAllFieldMapper allFieldMapper; + private final JsonObjectMapper rootObjectMapper; private final Analyzer indexAnalyzer; @@ -165,6 +174,7 @@ public class JsonDocumentMapper implements DocumentMapper, ToJson { JsonIdFieldMapper idFieldMapper, JsonTypeFieldMapper typeFieldMapper, JsonSourceFieldMapper sourceFieldMapper, + JsonAllFieldMapper allFieldMapper, Analyzer indexAnalyzer, Analyzer searchAnalyzer, @Nullable JsonBoostFieldMapper boostFieldMapper, @Nullable String mappingSource) { @@ -175,11 +185,17 @@ public class JsonDocumentMapper implements DocumentMapper, ToJson { this.idFieldMapper = idFieldMapper; this.typeFieldMapper = typeFieldMapper; this.sourceFieldMapper = sourceFieldMapper; + this.allFieldMapper = allFieldMapper; this.boostFieldMapper = boostFieldMapper; this.indexAnalyzer = indexAnalyzer; this.searchAnalyzer = searchAnalyzer; + // if we are not enabling all, set it to false on the root object, (and on all the rest...) + if (!allFieldMapper.enabled()) { + this.rootObjectMapper.includeInAll(allFieldMapper.enabled()); + } + rootObjectMapper.putMapper(idFieldMapper); if (boostFieldMapper != null) { rootObjectMapper.putMapper(boostFieldMapper); @@ -235,6 +251,10 @@ public class JsonDocumentMapper implements DocumentMapper, ToJson { return this.boostFieldMapper; } + @Override public AllFieldMapper allFieldMapper() { + return this.allFieldMapper; + } + @Override public Analyzer indexAnalyzer() { return this.indexAnalyzer; } @@ -311,6 +331,7 @@ public class JsonDocumentMapper implements DocumentMapper, ToJson { jsonContext.parsedId(JsonParseContext.ParsedIdState.EXTERNAL); idFieldMapper.parse(jsonContext); } + allFieldMapper.parse(jsonContext); } catch (IOException e) { throw new MapperParsingException("Failed to parse", e); } finally { @@ -342,6 +363,7 @@ public class JsonDocumentMapper implements DocumentMapper, ToJson { fieldMapperListener.fieldMapper(typeFieldMapper); fieldMapperListener.fieldMapper(idFieldMapper); fieldMapperListener.fieldMapper(uidFieldMapper); + fieldMapperListener.fieldMapper(allFieldMapper); rootObjectMapper.traverse(fieldMapperListener); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDocumentMapperParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDocumentMapperParser.java index bbea50b73cf..5dd2705001e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDocumentMapperParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDocumentMapperParser.java @@ -103,6 +103,8 @@ public class JsonDocumentMapperParser implements DocumentMapperParser { docBuilder.uidField(parseUidField((ObjectNode) fieldNode)); } else if (JsonBoostFieldMapper.JSON_TYPE.equals(fieldName)) { docBuilder.boostField(parseBoostField((ObjectNode) fieldNode)); + } else if (JsonAllFieldMapper.JSON_TYPE.equals(fieldName)) { + docBuilder.allField(parseAllField((ObjectNode) fieldNode)); } else if ("indexAnalyzer".equals(fieldName)) { docBuilder.indexAnalyzer(analysisService.analyzer(fieldNode.getTextValue())); } else if ("searchAnalyzer".equals(fieldName)) { @@ -173,6 +175,21 @@ public class JsonDocumentMapperParser implements DocumentMapperParser { return builder; } + private JsonAllFieldMapper.Builder parseAllField(ObjectNode allNode) { +// String name = idNode.get("name") == null ? JsonIdFieldMapper.Defaults.NAME : idNode.get("name").getTextValue(); + JsonAllFieldMapper.Builder builder = all(); + parseJsonField(builder, builder.name, allNode); + for (Iterator> fieldsIt = allNode.getFields(); fieldsIt.hasNext();) { + Map.Entry entry = fieldsIt.next(); + String fieldName = entry.getKey(); + JsonNode fieldNode = entry.getValue(); + if (fieldName.equals("enabled")) { + builder.enabled(nodeBooleanValue(fieldNode)); + } + } + return builder; + } + private JsonSourceFieldMapper.Builder parseSourceField(ObjectNode sourceNode) { // String name = sourceNode.get("name") == null ? JsonSourceFieldMapper.Defaults.NAME : sourceNode.get("name").getTextValue(); JsonSourceFieldMapper.Builder builder = source(); @@ -233,6 +250,8 @@ public class JsonDocumentMapperParser implements DocumentMapperParser { builder.pathType(parsePathType(name, fieldNode.getValueAsText())); } else if (fieldName.equals("properties")) { parseProperties(builder, (ObjectNode) fieldNode); + } else if (fieldName.equals("includeInAll")) { + builder.includeInAll(nodeBooleanValue(fieldNode)); } } return builder; @@ -483,6 +502,8 @@ public class JsonDocumentMapperParser implements DocumentMapperParser { } else if (propName.equals("analyzer")) { builder.indexAnalyzer(analysisService.analyzer(propNode.getTextValue())); builder.searchAnalyzer(analysisService.analyzer(propNode.getTextValue())); + } else if (propName.equals("includeInAll")) { + builder.includeInAll(nodeBooleanValue(propNode)); } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDoubleFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDoubleFieldMapper.java index 424419a6410..554fe669ea4 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDoubleFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDoubleFieldMapper.java @@ -32,7 +32,7 @@ import org.elasticsearch.util.json.JsonBuilder; import java.io.IOException; /** - * @author kimchy (Shay Banon) + * @author kimchy (shay.banon) */ public class JsonDoubleFieldMapper extends JsonNumberFieldMapper { @@ -57,13 +57,17 @@ public class JsonDoubleFieldMapper extends JsonNumberFieldMapper { } @Override public JsonDoubleFieldMapper build(BuilderContext context) { - return new JsonDoubleFieldMapper(buildNames(context), + JsonDoubleFieldMapper fieldMapper = new JsonDoubleFieldMapper(buildNames(context), precisionStep, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue); + fieldMapper.includeInAll(includeInAll); + return fieldMapper; } } private final Double nullValue; + private final String nullValueAsString; + protected JsonDoubleFieldMapper(Names names, int precisionStep, Field.Index index, Field.Store store, float boost, boolean omitNorms, boolean omitTermFreqAndPositions, @@ -72,6 +76,7 @@ public class JsonDoubleFieldMapper extends JsonNumberFieldMapper { new NamedAnalyzer("_double/" + precisionStep, new NumericDoubleAnalyzer(precisionStep)), new NamedAnalyzer("_double/max", new NumericDoubleAnalyzer(Integer.MAX_VALUE))); this.nullValue = nullValue; + this.nullValueAsString = nullValue == null ? null : nullValue.toString(); } @Override protected int maxPrecisionStep() { @@ -115,12 +120,18 @@ public class JsonDoubleFieldMapper extends JsonNumberFieldMapper { return null; } value = nullValue; + if (includeInAll == null || includeInAll) { + jsonContext.allEntries().addText(names.fullName(), nullValueAsString, boost); + } } else { if (jsonContext.jp().getCurrentToken() == JsonToken.VALUE_STRING) { value = Double.parseDouble(jsonContext.jp().getText()); } else { value = jsonContext.jp().getDoubleValue(); } + if (includeInAll == null || includeInAll) { + jsonContext.allEntries().addText(names.fullName(), jsonContext.jp().getText(), boost); + } } Field field = null; if (stored()) { @@ -147,5 +158,8 @@ public class JsonDoubleFieldMapper extends JsonNumberFieldMapper { if (nullValue != null) { builder.field("nullValue", nullValue); } + if (includeInAll != null) { + builder.field("includeInAll", includeInAll); + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonFieldMapper.java index f8052fcaead..431f89b6ea3 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonFieldMapper.java @@ -110,6 +110,8 @@ public abstract class JsonFieldMapper implements FieldMapper, JsonMapper { protected NamedAnalyzer searchAnalyzer; + protected Boolean includeInAll; + protected Builder(String name) { super(name); } @@ -162,6 +164,11 @@ public abstract class JsonFieldMapper implements FieldMapper, JsonMapper { return builder; } + protected T includeInAll(Boolean includeInAll) { + this.includeInAll = includeInAll; + return builder; + } + protected Names buildNames(BuilderContext context) { return new Names(name, buildIndexName(context), indexName == null ? name : indexName, buildFullName(context)); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonFloatFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonFloatFieldMapper.java index f312c3fe764..865ba811295 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonFloatFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonFloatFieldMapper.java @@ -57,14 +57,18 @@ public class JsonFloatFieldMapper extends JsonNumberFieldMapper { } @Override public JsonFloatFieldMapper build(BuilderContext context) { - return new JsonFloatFieldMapper(buildNames(context), + JsonFloatFieldMapper fieldMapper = new JsonFloatFieldMapper(buildNames(context), precisionStep, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue); + fieldMapper.includeInAll(includeInAll); + return fieldMapper; } } private final Float nullValue; + private final String nullValueAsString; + protected JsonFloatFieldMapper(Names names, int precisionStep, Field.Index index, Field.Store store, float boost, boolean omitNorms, boolean omitTermFreqAndPositions, Float nullValue) { @@ -72,6 +76,7 @@ public class JsonFloatFieldMapper extends JsonNumberFieldMapper { new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)), new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE))); this.nullValue = nullValue; + this.nullValueAsString = nullValue == null ? null : nullValue.toString(); } @Override protected int maxPrecisionStep() { @@ -115,12 +120,18 @@ public class JsonFloatFieldMapper extends JsonNumberFieldMapper { return null; } value = nullValue; + if (includeInAll == null || includeInAll) { + jsonContext.allEntries().addText(names.fullName(), nullValueAsString, boost); + } } else { if (jsonContext.jp().getCurrentToken() == JsonToken.VALUE_STRING) { value = Float.parseFloat(jsonContext.jp().getText()); } else { value = jsonContext.jp().getFloatValue(); } + if (includeInAll == null || includeInAll) { + jsonContext.allEntries().addText(names.fullName(), jsonContext.jp().getText(), boost); + } } Field field = null; if (stored()) { @@ -147,5 +158,8 @@ public class JsonFloatFieldMapper extends JsonNumberFieldMapper { if (nullValue != null) { builder.field("nullValue", nullValue); } + if (includeInAll != null) { + builder.field("includeInAll", includeInAll); + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIdFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIdFieldMapper.java index 4272755c571..004f729ddd1 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIdFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIdFieldMapper.java @@ -22,7 +22,6 @@ package org.elasticsearch.index.mapper.json; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; -import org.elasticsearch.index.mapper.FieldMapperListener; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MergeMappingException; @@ -72,8 +71,8 @@ public class JsonIdFieldMapper extends JsonFieldMapper implements IdFiel Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS); } - public JsonIdFieldMapper(String name, String indexName, Field.Store store, Field.TermVector termVector, - float boost, boolean omitNorms, boolean omitTermFreqAndPositions) { + protected JsonIdFieldMapper(String name, String indexName, Field.Store store, Field.TermVector termVector, + float boost, boolean omitNorms, boolean omitTermFreqAndPositions) { super(new Names(name, indexName, indexName, name), Defaults.INDEX, store, termVector, boost, omitNorms, omitTermFreqAndPositions, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER); } @@ -114,10 +113,6 @@ public class JsonIdFieldMapper extends JsonFieldMapper implements IdFiel } } - @Override public void traverse(FieldMapperListener fieldMapperListener) { - fieldMapperListener.fieldMapper(this); - } - @Override protected String jsonType() { return JSON_TYPE; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIncludeInAllMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIncludeInAllMapper.java new file mode 100644 index 00000000000..b64d2d2691a --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIncludeInAllMapper.java @@ -0,0 +1,28 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.json; + +/** + * @author kimchy (shay.banon) + */ +public interface JsonIncludeInAllMapper extends JsonMapper { + + void includeInAll(Boolean includeInAll); +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIntegerFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIntegerFieldMapper.java index 5e2a63c39e7..e06407d5546 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIntegerFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonIntegerFieldMapper.java @@ -32,7 +32,7 @@ import org.elasticsearch.util.json.JsonBuilder; import java.io.IOException; /** - * @author kimchy (Shay Banon) + * @author kimchy (shay.banon) */ public class JsonIntegerFieldMapper extends JsonNumberFieldMapper { @@ -57,13 +57,17 @@ public class JsonIntegerFieldMapper extends JsonNumberFieldMapper { } @Override public JsonIntegerFieldMapper build(BuilderContext context) { - return new JsonIntegerFieldMapper(buildNames(context), + JsonIntegerFieldMapper fieldMapper = new JsonIntegerFieldMapper(buildNames(context), precisionStep, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue); + fieldMapper.includeInAll(includeInAll); + return fieldMapper; } } private final Integer nullValue; + private final String nullValueAsString; + protected JsonIntegerFieldMapper(Names names, int precisionStep, Field.Index index, Field.Store store, float boost, boolean omitNorms, boolean omitTermFreqAndPositions, Integer nullValue) { @@ -71,6 +75,7 @@ public class JsonIntegerFieldMapper extends JsonNumberFieldMapper { new NamedAnalyzer("_int/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), new NamedAnalyzer("_int/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE))); this.nullValue = nullValue; + this.nullValueAsString = nullValue == null ? null : nullValue.toString(); } @Override protected int maxPrecisionStep() { @@ -114,12 +119,18 @@ public class JsonIntegerFieldMapper extends JsonNumberFieldMapper { return null; } value = nullValue; + if (includeInAll == null || includeInAll) { + jsonContext.allEntries().addText(names.fullName(), nullValueAsString, boost); + } } else { if (jsonContext.jp().getCurrentToken() == JsonToken.VALUE_STRING) { value = Integer.parseInt(jsonContext.jp().getText()); } else { value = jsonContext.jp().getIntValue(); } + if (includeInAll == null || includeInAll) { + jsonContext.allEntries().addText(names.fullName(), jsonContext.jp().getText(), boost); + } } Field field = null; if (stored()) { @@ -146,5 +157,8 @@ public class JsonIntegerFieldMapper extends JsonNumberFieldMapper { if (nullValue != null) { builder.field("nullValue", nullValue); } + if (includeInAll != null) { + builder.field("includeInAll", includeInAll); + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonLongFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonLongFieldMapper.java index b8b1579b0a9..dc3076873e0 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonLongFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonLongFieldMapper.java @@ -32,7 +32,7 @@ import org.elasticsearch.util.json.JsonBuilder; import java.io.IOException; /** - * @author kimchy (Shay Banon) + * @author kimchy (shay.banon) */ public class JsonLongFieldMapper extends JsonNumberFieldMapper { @@ -57,13 +57,17 @@ public class JsonLongFieldMapper extends JsonNumberFieldMapper { } @Override public JsonLongFieldMapper build(BuilderContext context) { - return new JsonLongFieldMapper(buildNames(context), + JsonLongFieldMapper fieldMapper = new JsonLongFieldMapper(buildNames(context), precisionStep, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue); + fieldMapper.includeInAll(includeInAll); + return fieldMapper; } } private final Long nullValue; + private final String nullValueAsString; + protected JsonLongFieldMapper(Names names, int precisionStep, Field.Index index, Field.Store store, float boost, boolean omitNorms, boolean omitTermFreqAndPositions, Long nullValue) { @@ -71,6 +75,7 @@ public class JsonLongFieldMapper extends JsonNumberFieldMapper { new NamedAnalyzer("_long/" + precisionStep, new NumericLongAnalyzer(precisionStep)), new NamedAnalyzer("_long/max", new NumericLongAnalyzer(Integer.MAX_VALUE))); this.nullValue = nullValue; + this.nullValueAsString = nullValue == null ? null : nullValue.toString(); } @Override protected int maxPrecisionStep() { @@ -114,12 +119,18 @@ public class JsonLongFieldMapper extends JsonNumberFieldMapper { return null; } value = nullValue; + if (includeInAll == null || includeInAll) { + jsonContext.allEntries().addText(names.fullName(), nullValueAsString, boost); + } } else { if (jsonContext.jp().getCurrentToken() == JsonToken.VALUE_STRING) { value = Long.parseLong(jsonContext.jp().getText()); } else { value = jsonContext.jp().getLongValue(); } + if (includeInAll == null || includeInAll) { + jsonContext.allEntries().addText(names.fullName(), jsonContext.jp().getText(), boost); + } } Field field = null; if (stored()) { @@ -146,5 +157,8 @@ public class JsonLongFieldMapper extends JsonNumberFieldMapper { if (nullValue != null) { builder.field("nullValue", nullValue); } + if (includeInAll != null) { + builder.field("includeInAll", includeInAll); + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMapperBuilders.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMapperBuilders.java index bed5dd47775..6be0f6f0ea5 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMapperBuilders.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMapperBuilders.java @@ -52,6 +52,10 @@ public final class JsonMapperBuilders { return new JsonBoostFieldMapper.Builder(name); } + public static JsonAllFieldMapper.Builder all() { + return new JsonAllFieldMapper.Builder(); + } + public static JsonMultiFieldMapper.Builder multiField(String name) { return new JsonMultiFieldMapper.Builder(name); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMultiFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMultiFieldMapper.java index f3cb9ff6046..d6e0bd1caa2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMultiFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonMultiFieldMapper.java @@ -36,7 +36,7 @@ import static org.elasticsearch.util.MapBuilder.*; /** * @author kimchy (shay.banon) */ -public class JsonMultiFieldMapper implements JsonMapper { +public class JsonMultiFieldMapper implements JsonMapper, JsonIncludeInAllMapper { public static final String JSON_TYPE = "multi_field"; @@ -113,12 +113,25 @@ public class JsonMultiFieldMapper implements JsonMapper { this.pathType = pathType; this.mappers = ImmutableMap.copyOf(mappers); this.defaultMapper = defaultMapper; + + // we disable the all in mappers, only the default one can be added + for (JsonMapper mapper : mappers.values()) { + if (mapper instanceof JsonIncludeInAllMapper) { + ((JsonIncludeInAllMapper) mapper).includeInAll(false); + } + } } @Override public String name() { return this.name; } + @Override public void includeInAll(Boolean includeInAll) { + if (includeInAll != null && defaultMapper != null && (defaultMapper instanceof JsonIncludeInAllMapper)) { + ((JsonIncludeInAllMapper) defaultMapper).includeInAll(includeInAll); + } + } + public JsonPath.Type pathType() { return pathType; } @@ -176,6 +189,10 @@ public class JsonMultiFieldMapper implements JsonMapper { if (mergeIntoMapper == null) { // no mapping, simply add it if not simulating if (!mergeContext.mergeFlags().simulate()) { + // disable the mapper from being in all, only the default mapper is in all + if (mergeWithMapper instanceof JsonIncludeInAllMapper) { + ((JsonIncludeInAllMapper) mergeWithMapper).includeInAll(false); + } mappers = newMapBuilder(mappers).put(mergeWithMapper.name(), mergeWithMapper).immutableMap(); if (mergeWithMapper instanceof JsonFieldMapper) { mergeContext.docMapper().addFieldMapper((FieldMapper) mergeWithMapper); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonNumberFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonNumberFieldMapper.java index 985fb702869..c5fd87e1c6d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonNumberFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonNumberFieldMapper.java @@ -33,9 +33,9 @@ import java.util.ArrayDeque; import java.util.Deque; /** - * @author kimchy (Shay Banon) + * @author kimchy (shay.banon) */ -public abstract class JsonNumberFieldMapper extends JsonFieldMapper { +public abstract class JsonNumberFieldMapper extends JsonFieldMapper implements JsonIncludeInAllMapper { public static class Defaults extends JsonFieldMapper.Defaults { public static final int PRECISION_STEP = NumericUtils.PRECISION_STEP_DEFAULT; @@ -59,14 +59,18 @@ public abstract class JsonNumberFieldMapper extends JsonFieldM return super.store(store); } - @Override protected T boost(float boost) { + @Override public T boost(float boost) { return super.boost(boost); } - @Override protected T indexName(String indexName) { + @Override public T indexName(String indexName) { return super.indexName(indexName); } + @Override public T includeInAll(Boolean includeInAll) { + return super.includeInAll(includeInAll); + } + public T precisionStep(int precisionStep) { this.precisionStep = precisionStep; return builder; @@ -81,6 +85,8 @@ public abstract class JsonNumberFieldMapper extends JsonFieldM protected final int precisionStep; + protected Boolean includeInAll; + protected JsonNumberFieldMapper(Names names, int precisionStep, Field.Index index, Field.Store store, float boost, boolean omitNorms, boolean omitTermFreqAndPositions, @@ -93,6 +99,12 @@ public abstract class JsonNumberFieldMapper extends JsonFieldM } } + @Override public void includeInAll(Boolean includeInAll) { + if (includeInAll != null) { + this.includeInAll = includeInAll; + } + } + protected abstract int maxPrecisionStep(); public int precisionStep() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonObjectMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonObjectMapper.java index d9e880134c9..91baad24514 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonObjectMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonObjectMapper.java @@ -42,10 +42,10 @@ import static org.elasticsearch.index.mapper.json.JsonMapperBuilders.*; import static org.elasticsearch.util.MapBuilder.*; /** - * @author kimchy (Shay Banon) + * @author kimchy (shay.banon) */ @ThreadSafe -public class JsonObjectMapper implements JsonMapper { +public class JsonObjectMapper implements JsonMapper, JsonIncludeInAllMapper { public static final String JSON_TYPE = "object"; @@ -66,6 +66,8 @@ public class JsonObjectMapper implements JsonMapper { private List dateTimeFormatters = newArrayList(); + private Boolean includeInAll; + private final List mappersBuilders = newArrayList(); public Builder(String name) { @@ -93,6 +95,11 @@ public class JsonObjectMapper implements JsonMapper { return this; } + public Builder includeInAll(boolean includeInAll) { + this.includeInAll = includeInAll; + return this; + } + public Builder dateTimeFormatter(Iterable dateTimeFormatters) { for (FormatDateTimeFormatter dateTimeFormatter : dateTimeFormatters) { this.dateTimeFormatters.add(dateTimeFormatter); @@ -138,6 +145,8 @@ public class JsonObjectMapper implements JsonMapper { context.path().pathType(origPathType); context.path().remove(); + objectMapper.includeInAll(includeInAll); + return objectMapper; } } @@ -152,6 +161,8 @@ public class JsonObjectMapper implements JsonMapper { private final FormatDateTimeFormatter[] dateTimeFormatters; + private Boolean includeInAll; + private volatile ImmutableMap mappers = ImmutableMap.of(); private final Object mutex = new Object(); @@ -185,7 +196,23 @@ public class JsonObjectMapper implements JsonMapper { return this.name; } + @Override public void includeInAll(Boolean includeInAll) { + if (includeInAll == null) { + return; + } + this.includeInAll = includeInAll; + // when called from outside, apply this on all the inner mappers + for (JsonMapper mapper : mappers.values()) { + if (mapper instanceof JsonIncludeInAllMapper) { + ((JsonIncludeInAllMapper) mapper).includeInAll(includeInAll); + } + } + } + public JsonObjectMapper putMapper(JsonMapper mapper) { + if (mapper instanceof JsonIncludeInAllMapper) { + ((JsonIncludeInAllMapper) mapper).includeInAll(includeInAll); + } synchronized (mutex) { mappers = newMapBuilder(mappers).put(mapper.name(), mapper).immutableMap(); } @@ -389,6 +416,9 @@ public class JsonObjectMapper implements JsonMapper { builder.field("dynamic", dynamic); builder.field("enabled", enabled); builder.field("pathType", pathType.name().toLowerCase()); + if (includeInAll != null) { + builder.field("includeInAll", includeInAll); + } if (dateTimeFormatters.length > 0) { builder.startArray("dateFormats"); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonParseContext.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonParseContext.java index bcf8fa9f720..1d78e28602c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonParseContext.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonParseContext.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.Document; import org.codehaus.jackson.JsonParser; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.util.concurrent.NotThreadSafe; +import org.elasticsearch.util.lucene.all.AllEntries; /** * @author kimchy (Shay Banon) @@ -54,6 +55,8 @@ public class JsonParseContext { private boolean mappersAdded = false; + private AllEntries allEntries = new AllEntries(); + public JsonParseContext(JsonDocumentMapper docMapper, JsonPath path) { this.docMapper = docMapper; this.path = path; @@ -68,6 +71,7 @@ public class JsonParseContext { this.parsedIdState = ParsedIdState.NO; this.mappersAdded = false; this.listener = listener; + this.allEntries = new AllEntries(); } public boolean mappersAdded() { @@ -136,6 +140,10 @@ public class JsonParseContext { this.uid = uid; } + public AllEntries allEntries() { + return this.allEntries; + } + /** * A string builder that can be used to construct complex names for example. * Its better to reuse the. diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonStringFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonStringFieldMapper.java index ca5ab1f95a0..0870ce8e78c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonStringFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonStringFieldMapper.java @@ -30,7 +30,7 @@ import java.io.IOException; /** * @author kimchy (shay.banon) */ -public class JsonStringFieldMapper extends JsonFieldMapper { +public class JsonStringFieldMapper extends JsonFieldMapper implements JsonIncludeInAllMapper { public static final String JSON_TYPE = "string"; @@ -53,15 +53,24 @@ public class JsonStringFieldMapper extends JsonFieldMapper { return this; } + @Override public Builder includeInAll(Boolean includeInAll) { + this.includeInAll = includeInAll; + return this; + } + @Override public JsonStringFieldMapper build(BuilderContext context) { - return new JsonStringFieldMapper(buildNames(context), + JsonStringFieldMapper fieldMapper = new JsonStringFieldMapper(buildNames(context), index, store, termVector, boost, omitNorms, omitTermFreqAndPositions, nullValue, indexAnalyzer, searchAnalyzer); + fieldMapper.includeInAll(includeInAll); + return fieldMapper; } } private final String nullValue; + private Boolean includeInAll; + protected JsonStringFieldMapper(Names names, Field.Index index, Field.Store store, Field.TermVector termVector, float boost, boolean omitNorms, boolean omitTermFreqAndPositions, String nullValue, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer) { @@ -69,6 +78,12 @@ public class JsonStringFieldMapper extends JsonFieldMapper { this.nullValue = nullValue; } + @Override public void includeInAll(Boolean includeInAll) { + if (includeInAll != null) { + this.includeInAll = includeInAll; + } + } + @Override public String value(Fieldable field) { return field.stringValue(); } @@ -91,6 +106,9 @@ public class JsonStringFieldMapper extends JsonFieldMapper { if (value == null) { return null; } + if (includeInAll == null || includeInAll) { + jsonContext.allEntries().addText(names.fullName(), value, boost); + } return new Field(names.indexName(), value, store, index, termVector); } @@ -103,5 +121,8 @@ public class JsonStringFieldMapper extends JsonFieldMapper { if (nullValue != null) { builder.field("nullValue", nullValue); } + if (includeInAll != null) { + builder.field("includeInAll", includeInAll); + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/JsonQueryBuilders.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/JsonQueryBuilders.java index 82cff056c69..b70e4151e4e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/JsonQueryBuilders.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/JsonQueryBuilders.java @@ -272,6 +272,14 @@ public abstract class JsonQueryBuilders { return new MoreLikeThisJsonQueryBuilder(fields); } + /** + * A more like this query that finds documents that are "like" the provided {@link MoreLikeThisJsonQueryBuilder#likeText(String)} + * which is checked against the "_all" field. + */ + public static MoreLikeThisJsonQueryBuilder moreLikeThisQuery() { + return new MoreLikeThisJsonQueryBuilder(); + } + /** * A more like this query that runs against a specific field. * diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/MoreLikeThisJsonQueryBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/MoreLikeThisJsonQueryBuilder.java index 67b340301bf..6518b686733 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/MoreLikeThisJsonQueryBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/MoreLikeThisJsonQueryBuilder.java @@ -46,6 +46,13 @@ public class MoreLikeThisJsonQueryBuilder extends BaseJsonQueryBuilder { private Boolean boostTerms = null; private float boostTermsFactor = -1; + /** + * Constructs a new more like this query which uses the "_all" field. + */ + public MoreLikeThisJsonQueryBuilder() { + this.fields = null; + } + /** * Sets the field names that will be used when generating the 'More Like This' query. * @@ -159,11 +166,13 @@ public class MoreLikeThisJsonQueryBuilder extends BaseJsonQueryBuilder { if (fields == null || fields.length == 0) { throw new QueryBuilderException("moreLikeThis requires 'fields' to be provided"); } - builder.startArray("fields"); - for (String field : fields) { - builder.value(field); + if (fields != null) { + builder.startArray("fields"); + for (String field : fields) { + builder.value(field); + } + builder.endArray(); } - builder.endArray(); if (likeText == null) { throw new QueryBuilderException("moreLikeThis requires 'likeText' to be provided"); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/MoreLikeThisJsonQueryParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/MoreLikeThisJsonQueryParser.java index c8953f4e242..4ada2548f05 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/MoreLikeThisJsonQueryParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/MoreLikeThisJsonQueryParser.java @@ -25,6 +25,7 @@ import org.codehaus.jackson.JsonParser; import org.codehaus.jackson.JsonToken; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; +import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.query.QueryParsingException; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.util.Booleans; @@ -56,6 +57,7 @@ public class MoreLikeThisJsonQueryParser extends AbstractIndexComponent implemen JsonParser jp = parseContext.jp(); MoreLikeThisQuery mltQuery = new MoreLikeThisQuery(); + mltQuery.setMoreLikeFields(new String[]{AllFieldMapper.NAME}); JsonToken token; String currentFieldName = null; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/QueryStringJsonQueryParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/QueryStringJsonQueryParser.java index 3ec0ff3e3c4..9f44ba3a5ed 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/QueryStringJsonQueryParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/json/QueryStringJsonQueryParser.java @@ -31,6 +31,7 @@ import org.codehaus.jackson.JsonToken; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.query.QueryParsingException; import org.elasticsearch.index.query.support.MapperQueryParser; import org.elasticsearch.index.query.support.MultiFieldMapperQueryParser; @@ -45,7 +46,7 @@ import java.util.List; import static org.elasticsearch.util.lucene.search.Queries.*; /** - * @author kimchy (Shay Banon) + * @author kimchy (shay.banon) */ public class QueryStringJsonQueryParser extends AbstractIndexComponent implements JsonQueryParser { @@ -68,7 +69,7 @@ public class QueryStringJsonQueryParser extends AbstractIndexComponent implement // move to the field value String queryString = null; - String defaultField = null; + String defaultField = AllFieldMapper.NAME; // default to all MapperQueryParser.Operator defaultOperator = QueryParser.Operator.OR; boolean allowLeadingWildcard = true; boolean lowercaseExpandedTerms = true; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllEntries.java b/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllEntries.java new file mode 100644 index 00000000000..26d6b992884 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllEntries.java @@ -0,0 +1,159 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.util.lucene.all; + +import com.google.common.collect.Lists; +import org.elasticsearch.util.io.FastStringReader; + +import java.io.IOException; +import java.io.Reader; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import static com.google.common.collect.Sets.*; + +/** + * @author kimchy (shay.banon) + */ +public class AllEntries extends Reader { + + public static class Entry { + private final String name; + private final Reader reader; + private final float boost; + + public Entry(String name, Reader reader, float boost) { + this.name = name; + this.reader = reader; + this.boost = boost; + } + + public String name() { + return this.name; + } + + public float boost() { + return this.boost; + } + + public Reader reader() { + return this.reader; + } + } + + private final List entries = Lists.newArrayList(); + + private Entry current; + + private Iterator it; + + private boolean itsSeparatorTime = false; + + public void addText(String name, String text, float boost) { + Entry entry = new Entry(name, new FastStringReader(text), boost); + entries.add(entry); + } + + public void clear() { + this.entries.clear(); + this.current = null; + this.it = null; + itsSeparatorTime = false; + } + + public AllEntries finishTexts() { + it = entries.iterator(); + if (it.hasNext()) { + current = it.next(); + itsSeparatorTime = true; + } + return this; + } + + public List entries() { + return this.entries; + } + + public Set fields() { + Set fields = newHashSet(); + for (Entry entry : entries) { + fields.add(entry.name()); + } + return fields; + } + + public Entry current() { + return this.current; + } + + @Override public int read(char[] cbuf, int off, int len) throws IOException { + if (current == null) { + return -1; + } + int result = current.reader().read(cbuf, off, len); + if (result == -1) { + if (itsSeparatorTime) { + itsSeparatorTime = false; + cbuf[off] = ' '; + return 1; + } + itsSeparatorTime = true; + advance(); + return read(cbuf, off, len); + } + return result; + } + + @Override public void close() { + if (current != null) { + try { + current.reader().close(); + } catch (IOException e) { + // can't happen... + } finally { + current = null; + } + } + } + + + @Override public boolean ready() throws IOException { + return (current != null) && current.reader().ready(); + } + + /** + * Closes the current reader and opens the next one, if any. + */ + private void advance() { + close(); + if (it.hasNext()) { + current = it.next(); + } + } + + @Override public String toString() { + StringBuilder sb = new StringBuilder(); + for (Entry entry : entries) { + sb.append(entry.name()).append(','); + } + return sb.toString(); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTermQuery.java b/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTermQuery.java new file mode 100644 index 00000000000..d97dc6d4af1 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTermQuery.java @@ -0,0 +1,188 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.util.lucene.all; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermPositions; +import org.apache.lucene.search.*; +import org.apache.lucene.search.spans.SpanScorer; +import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.search.spans.SpanWeight; +import org.apache.lucene.search.spans.TermSpans; + +import java.io.IOException; + +import static org.apache.lucene.analysis.payloads.PayloadHelper.*; + +/** + * @author kimchy (shay.banon) + */ +public class AllTermQuery extends SpanTermQuery { + + private boolean includeSpanScore; + + public AllTermQuery(Term term) { + this(term, true); + } + + public AllTermQuery(Term term, boolean includeSpanScore) { + super(term); + this.includeSpanScore = includeSpanScore; + } + + @Override + public Weight createWeight(Searcher searcher) throws IOException { + return new AllTermWeight(this, searcher); + } + + protected class AllTermWeight extends SpanWeight { + + public AllTermWeight(AllTermQuery query, Searcher searcher) throws IOException { + super(query, searcher); + } + + @Override + public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, + boolean topScorer) throws IOException { + return new AllTermSpanScorer((TermSpans) query.getSpans(reader), this, similarity, reader.norms(query.getField())); + } + + protected class AllTermSpanScorer extends SpanScorer { + // TODO: is this the best way to allocate this? + protected byte[] payload = new byte[4]; + protected TermPositions positions; + protected float payloadScore; + protected int payloadsSeen; + + public AllTermSpanScorer(TermSpans spans, Weight weight, Similarity similarity, byte[] norms) throws IOException { + super(spans, weight, similarity, norms); + positions = spans.getPositions(); + } + + @Override + protected boolean setFreqCurrentDoc() throws IOException { + if (!more) { + return false; + } + doc = spans.doc(); + freq = 0.0f; + payloadScore = 0; + payloadsSeen = 0; + Similarity similarity1 = getSimilarity(); + while (more && doc == spans.doc()) { + int matchLength = spans.end() - spans.start(); + + freq += similarity1.sloppyFreq(matchLength); + processPayload(similarity1); + + more = spans.next();// this moves positions to the next match in this + // document + } + return more || (freq != 0); + } + + protected void processPayload(Similarity similarity) throws IOException { + if (positions.isPayloadAvailable()) { + payload = positions.getPayload(payload, 0); + payloadScore += decodeFloat(payload); + payloadsSeen++; + + } else { + // zero out the payload? + } + } + + /** + * @return {@link #getSpanScore()} * {@link #getPayloadScore()} + * @throws IOException + */ + @Override + public float score() throws IOException { + return includeSpanScore ? getSpanScore() * getPayloadScore() : getPayloadScore(); + } + + /** + * Returns the SpanScorer score only. + *

+ * Should not be overridden without good cause! + * + * @return the score for just the Span part w/o the payload + * @throws IOException + * @see #score() + */ + protected float getSpanScore() throws IOException { + return super.score(); + } + + /** + * The score for the payload + */ + protected float getPayloadScore() { + return payloadsSeen > 0 ? (payloadScore / payloadsSeen) : 1; + } + + @Override + protected Explanation explain(final int doc) throws IOException { + ComplexExplanation result = new ComplexExplanation(); + Explanation nonPayloadExpl = super.explain(doc); + result.addDetail(nonPayloadExpl); + // QUESTION: Is there a way to avoid this skipTo call? We need to know + // whether to load the payload or not + Explanation payloadBoost = new Explanation(); + result.addDetail(payloadBoost); + + float payloadScore = getPayloadScore(); + payloadBoost.setValue(payloadScore); + // GSI: I suppose we could toString the payload, but I don't think that + // would be a good idea + payloadBoost.setDescription("allPayload(...)"); + result.setValue(nonPayloadExpl.getValue() * payloadScore); + result.setDescription("btq, product of:"); + result.setMatch(nonPayloadExpl.getValue() == 0 ? Boolean.FALSE : Boolean.TRUE); // LUCENE-1303 + return result; + } + + } + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + (includeSpanScore ? 1231 : 1237); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + if (getClass() != obj.getClass()) + return false; + AllTermQuery other = (AllTermQuery) obj; + if (includeSpanScore != other.includeSpanScore) + return false; + return true; + } + +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenFilter.java new file mode 100644 index 00000000000..f4fe93c1c6e --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenFilter.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.util.lucene.all; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.index.Payload; + +import java.io.IOException; + +import static org.apache.lucene.analysis.payloads.PayloadHelper.*; + +/** + * @author kimchy (shay.banon) + */ +public class AllTokenFilter extends TokenFilter { + + public static TokenStream allTokenStream(String allFieldName, AllEntries allEntries, Analyzer analyzer) throws IOException { + return new AllTokenFilter(analyzer.reusableTokenStream(allFieldName, allEntries), allEntries); + } + + private final AllEntries allEntries; + + private final PayloadAttribute payloadAttribute; + + AllTokenFilter(TokenStream input, AllEntries allEntries) { + super(input); + this.allEntries = allEntries; + payloadAttribute = addAttribute(PayloadAttribute.class); + } + + public AllEntries allEntries() { + return allEntries; + } + + @Override public boolean incrementToken() throws IOException { + if (!input.incrementToken()) { + return false; + } + float boost = allEntries.current().boost(); + if (boost != 1.0f) { + payloadAttribute.setPayload(new Payload(encodeFloat(boost))); + } else { + payloadAttribute.setPayload(null); + } + return true; + } +} diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/SimpleAllMapperTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/SimpleAllMapperTests.java new file mode 100644 index 00000000000..878704df019 --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/SimpleAllMapperTests.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.json.all; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.mapper.json.JsonDocumentMapper; +import org.elasticsearch.index.mapper.json.JsonDocumentMapperParser; +import org.elasticsearch.util.lucene.all.AllEntries; +import org.elasticsearch.util.lucene.all.AllTokenFilter; +import org.hamcrest.MatcherAssert; +import org.testng.annotations.Test; + +import static org.elasticsearch.util.io.Streams.*; +import static org.hamcrest.Matchers.*; + +/** + * @author kimchy (shay.banon) + */ +@Test +public class SimpleAllMapperTests { + + @Test public void testSimpleAllMappers() throws Exception { + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/json/all/mapping.json"); + JsonDocumentMapper docMapper = (JsonDocumentMapper) new JsonDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping); + byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/json/all/test1.json"); + Document doc = docMapper.parse(json).doc(); + Field field = doc.getField("_all"); + AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries(); + MatcherAssert.assertThat(allEntries.fields().size(), equalTo(2)); + MatcherAssert.assertThat(allEntries.fields().contains("name.last"), equalTo(true)); + MatcherAssert.assertThat(allEntries.fields().contains("simple1"), equalTo(true)); + } + + @Test public void testSimpleAllMappersWithReparse() throws Exception { + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/json/all/mapping.json"); + JsonDocumentMapper docMapper = (JsonDocumentMapper) new JsonDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping); + String builtMapping = docMapper.buildSource(); +// System.out.println(builtMapping); + // reparse it + JsonDocumentMapper builtDocMapper = (JsonDocumentMapper) new JsonDocumentMapperParser(new AnalysisService(new Index("test"))).parse(builtMapping); + byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/json/all/test1.json"); + Document doc = builtDocMapper.parse(json).doc(); + + Field field = doc.getField("_all"); + AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries(); + MatcherAssert.assertThat(allEntries.fields().size(), equalTo(2)); + MatcherAssert.assertThat(allEntries.fields().contains("name.last"), equalTo(true)); + MatcherAssert.assertThat(allEntries.fields().contains("simple1"), equalTo(true)); + } +} diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/mapping.json b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/mapping.json new file mode 100644 index 00000000000..7ba46d2a703 --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/mapping.json @@ -0,0 +1,33 @@ +{ + person : { + allField : {enabled : true}, + properties : { + name : { + type : "object", + dynamic : false, + properties : { + first : {type : "string", store : "yes", includeInAll : false}, + last : {type : "string", index : "not_analyzed"} + } + }, + address : { + type : "object", + includeInAll : false, + properties : { + first : { + properties : { + location : {type : "string", store : "yes", indexName : "firstLocation"} + } + }, + last : { + properties : { + location : {type : "string"} + } + } + } + }, + simple1 : {type : "long", includeInAll : true}, + simple2 : {type : "long", includeInAll : false} + } + } +} \ No newline at end of file diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/test1.json b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/test1.json new file mode 100644 index 00000000000..8c5e43c1c5b --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/json/all/test1.json @@ -0,0 +1,20 @@ +{ + person : { + _boost : 3.7, + _id : "1", + name : { + first : "shay", + last : "banon" + }, + address : { + first : { + location : "first location" + }, + last : { + location : "last location" + } + }, + simple1 : 1, + simple2 : 2 + } +} \ No newline at end of file diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/util/lucene/all/SimpleAllTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/util/lucene/all/SimpleAllTests.java new file mode 100644 index 00000000000..bd2495a844e --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/util/lucene/all/SimpleAllTests.java @@ -0,0 +1,231 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.util.lucene.all; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.elasticsearch.util.lucene.Lucene; +import org.testng.annotations.Test; + +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +/** + * @author kimchy (shay.banon) + */ +@Test +public class SimpleAllTests { + + @Test public void testSimpleAllNoBoost() throws Exception { + Directory dir = new RAMDirectory(); + IndexWriter indexWriter = new IndexWriter(dir, Lucene.STANDARD_ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED); + + Document doc = new Document(); + doc.add(new Field("_id", "1", Field.Store.YES, Field.Index.NO)); + AllEntries allEntries = new AllEntries(); + allEntries.addText("field1", "something", 1.0f); + allEntries.addText("field2", "else", 1.0f); + allEntries.finishTexts(); + doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + + indexWriter.addDocument(doc); + + doc = new Document(); + doc.add(new Field("_id", "2", Field.Store.YES, Field.Index.NO)); + allEntries = new AllEntries(); + allEntries.addText("field1", "else", 1.0f); + allEntries.addText("field2", "something", 1.0f); + allEntries.finishTexts(); + doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + + indexWriter.addDocument(doc); + + IndexReader reader = indexWriter.getReader(); + IndexSearcher searcher = new IndexSearcher(reader); + + TopDocs docs = searcher.search(new AllTermQuery(new Term("_all", "else")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(0)); + assertThat(docs.scoreDocs[1].doc, equalTo(1)); + + docs = searcher.search(new AllTermQuery(new Term("_all", "something")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(0)); + assertThat(docs.scoreDocs[1].doc, equalTo(1)); + + searcher.close(); + + indexWriter.close(); + } + + @Test public void testSimpleAllWithBoost() throws Exception { + Directory dir = new RAMDirectory(); + IndexWriter indexWriter = new IndexWriter(dir, Lucene.STANDARD_ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED); + + Document doc = new Document(); + doc.add(new Field("_id", "1", Field.Store.YES, Field.Index.NO)); + AllEntries allEntries = new AllEntries(); + allEntries.addText("field1", "something", 1.0f); + allEntries.addText("field2", "else", 1.0f); + allEntries.finishTexts(); + doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + + indexWriter.addDocument(doc); + + doc = new Document(); + doc.add(new Field("_id", "2", Field.Store.YES, Field.Index.NO)); + allEntries = new AllEntries(); + allEntries.addText("field1", "else", 2.0f); + allEntries.addText("field2", "something", 1.0f); + allEntries.finishTexts(); + doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + + indexWriter.addDocument(doc); + + IndexReader reader = indexWriter.getReader(); + IndexSearcher searcher = new IndexSearcher(reader); + + // this one is boosted. so the second doc is more relevant + TopDocs docs = searcher.search(new AllTermQuery(new Term("_all", "else")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(1)); + assertThat(docs.scoreDocs[1].doc, equalTo(0)); + + docs = searcher.search(new AllTermQuery(new Term("_all", "something")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(0)); + assertThat(docs.scoreDocs[1].doc, equalTo(1)); + + searcher.close(); + + indexWriter.close(); + } + + @Test public void testMultipleTokensAllNoBoost() throws Exception { + Directory dir = new RAMDirectory(); + IndexWriter indexWriter = new IndexWriter(dir, Lucene.STANDARD_ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED); + + Document doc = new Document(); + doc.add(new Field("_id", "1", Field.Store.YES, Field.Index.NO)); + AllEntries allEntries = new AllEntries(); + allEntries.addText("field1", "something moo", 1.0f); + allEntries.addText("field2", "else koo", 1.0f); + allEntries.finishTexts(); + doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + + indexWriter.addDocument(doc); + + doc = new Document(); + doc.add(new Field("_id", "2", Field.Store.YES, Field.Index.NO)); + allEntries = new AllEntries(); + allEntries.addText("field1", "else koo", 1.0f); + allEntries.addText("field2", "something moo", 1.0f); + allEntries.finishTexts(); + doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + + indexWriter.addDocument(doc); + + IndexReader reader = indexWriter.getReader(); + IndexSearcher searcher = new IndexSearcher(reader); + + TopDocs docs = searcher.search(new AllTermQuery(new Term("_all", "else")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(0)); + assertThat(docs.scoreDocs[1].doc, equalTo(1)); + + docs = searcher.search(new AllTermQuery(new Term("_all", "koo")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(0)); + assertThat(docs.scoreDocs[1].doc, equalTo(1)); + + docs = searcher.search(new AllTermQuery(new Term("_all", "something")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(0)); + assertThat(docs.scoreDocs[1].doc, equalTo(1)); + + docs = searcher.search(new AllTermQuery(new Term("_all", "moo")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(0)); + assertThat(docs.scoreDocs[1].doc, equalTo(1)); + + searcher.close(); + + indexWriter.close(); + } + + @Test public void testMultipleTokensAllWithBoost() throws Exception { + Directory dir = new RAMDirectory(); + IndexWriter indexWriter = new IndexWriter(dir, Lucene.STANDARD_ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED); + + Document doc = new Document(); + doc.add(new Field("_id", "1", Field.Store.YES, Field.Index.NO)); + AllEntries allEntries = new AllEntries(); + allEntries.addText("field1", "something moo", 1.0f); + allEntries.addText("field2", "else koo", 1.0f); + allEntries.finishTexts(); + doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + + indexWriter.addDocument(doc); + + doc = new Document(); + doc.add(new Field("_id", "2", Field.Store.YES, Field.Index.NO)); + allEntries = new AllEntries(); + allEntries.addText("field1", "else koo", 2.0f); + allEntries.addText("field2", "something moo", 1.0f); + allEntries.finishTexts(); + doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + + indexWriter.addDocument(doc); + + IndexReader reader = indexWriter.getReader(); + IndexSearcher searcher = new IndexSearcher(reader); + + TopDocs docs = searcher.search(new AllTermQuery(new Term("_all", "else")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(1)); + assertThat(docs.scoreDocs[1].doc, equalTo(0)); + + docs = searcher.search(new AllTermQuery(new Term("_all", "koo")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(1)); + assertThat(docs.scoreDocs[1].doc, equalTo(0)); + + docs = searcher.search(new AllTermQuery(new Term("_all", "something")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(0)); + assertThat(docs.scoreDocs[1].doc, equalTo(1)); + + docs = searcher.search(new AllTermQuery(new Term("_all", "moo")), 10); + assertThat(docs.totalHits, equalTo(2)); + assertThat(docs.scoreDocs[0].doc, equalTo(0)); + assertThat(docs.scoreDocs[1].doc, equalTo(1)); + + searcher.close(); + + indexWriter.close(); + } +} diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/terms/TermsActionTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/terms/TermsActionTests.java index cbca813dc2d..c3b05708c54 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/terms/TermsActionTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/terms/TermsActionTests.java @@ -111,6 +111,17 @@ public class TermsActionTests extends AbstractServersTests { assertThat(termsResponse.field("value").docFreq("aaa"), equalTo(1)); assertThat(termsResponse.field("value").docFreq("bbb"), equalTo(1)); + logger.info("Verify freqs (no fields, on _all)"); + termsResponse = client.terms(termsRequest("test")).actionGet(); + assertThat(termsResponse.numDocs(), equalTo(2l)); + assertThat(termsResponse.maxDoc(), equalTo(2l)); + assertThat(termsResponse.deletedDocs(), equalTo(0l)); + assertThat(termsResponse.successfulShards(), equalTo(indexStatus.shards().size())); + assertThat(termsResponse.failedShards(), equalTo(0)); + assertThat(termsResponse.fieldsAsMap().isEmpty(), equalTo(false)); + assertThat(termsResponse.field("_all").docFreq("aaa"), equalTo(1)); + assertThat(termsResponse.field("_all").docFreq("bbb"), equalTo(1)); + logger.info("Delete 3"); client.index(indexRequest("test").type("type1").id("3").source(binaryJsonBuilder().startObject().field("value", "bbb").endObject())).actionGet(); logger.info("Refresh");