From ea9a4d70cf7140d6cf5c3c12e59fca0718164f4a Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 6 Dec 2012 18:18:20 +0100 Subject: [PATCH] lucene 4: Removed the usage of Document & Field when retrieving stored fields. --- .../mlt/TransportMoreLikeThisAction.java | 24 ++- .../AbstractMultipleFieldsVisitor.java | 63 ------- .../lucene/document/BaseFieldVisitor.java | 14 -- .../lucene/document/SingleFieldVisitor.java | 55 ------ .../fieldvisitor/CustomFieldsVisitor.java | 63 +++++++ .../index/fieldvisitor/FieldsVisitor.java | 151 ++++++++++++++++ .../fieldvisitor/JustSourceFieldsVisitor.java | 38 ++++ .../fieldvisitor/JustUidFieldsVisitor.java | 38 ++++ .../fieldvisitor/SingleFieldsVisitor.java | 65 +++++++ .../UidAndRoutingFieldsVisitor.java | 57 ++++++ .../UidAndSourceFieldsVisitor.java | 42 +++++ .../index/get/ShardGetService.java | 86 ++------- .../index/mapper/FieldMapper.java | 7 +- .../mapper/core/AbstractFieldMapper.java | 9 +- .../index/mapper/core/BinaryFieldMapper.java | 12 +- .../index/mapper/core/BooleanFieldMapper.java | 8 +- .../index/mapper/core/ByteFieldMapper.java | 10 +- .../index/mapper/core/DateFieldMapper.java | 23 ++- .../index/mapper/core/DoubleFieldMapper.java | 9 +- .../index/mapper/core/FloatFieldMapper.java | 9 +- .../index/mapper/core/IntegerFieldMapper.java | 9 +- .../index/mapper/core/LongFieldMapper.java | 9 +- .../index/mapper/core/NumberFieldMapper.java | 10 +- .../index/mapper/core/ShortFieldMapper.java | 9 +- .../index/mapper/core/StringFieldMapper.java | 9 +- .../index/mapper/geo/GeoShapeFieldMapper.java | 4 +- .../index/mapper/internal/AllFieldMapper.java | 6 +- .../mapper/internal/BoostFieldMapper.java | 5 +- .../index/mapper/internal/IdFieldMapper.java | 9 +- .../mapper/internal/IndexFieldMapper.java | 9 +- .../mapper/internal/ParentFieldMapper.java | 13 +- .../mapper/internal/RoutingFieldMapper.java | 9 +- .../mapper/internal/SourceFieldMapper.java | 18 +- .../mapper/internal/SourceFieldVisitor.java | 72 -------- .../index/mapper/internal/TTLFieldMapper.java | 6 +- .../mapper/internal/TimestampFieldMapper.java | 14 +- .../mapper/internal/TypeFieldMapper.java | 9 +- .../index/mapper/internal/UidFieldMapper.java | 9 +- .../index/mapper/ip/IpFieldMapper.java | 19 +- .../selector/AllButSourceFieldVisitor.java | 45 ----- .../selector/FieldMappersFieldVisitor.java | 57 ------ .../selector/UidAndRoutingFieldVisitor.java | 90 ---------- .../selector/UidAndSourceFieldVisitor.java | 83 --------- .../mapper/selector/UidFieldVisitor.java | 75 -------- .../index/percolator/PercolatorService.java | 18 +- .../indices/ttl/IndicesTTLService.java | 19 +- .../search/fetch/FetchPhase.java | 166 ++++-------------- .../search/fetch/FetchSubPhase.java | 12 +- .../fetch/version/VersionFetchSubPhase.java | 5 +- .../search/highlight/HighlightPhase.java | 15 +- .../search/internal/InternalSearchHit.java | 4 +- .../search/lookup/FieldLookup.java | 33 ++-- .../search/lookup/FieldsLookup.java | 17 +- .../search/lookup/SourceLookup.java | 9 +- .../test/integration/get/GetActionTests.java | 96 ++++++++++ 55 files changed, 811 insertions(+), 964 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/common/lucene/document/AbstractMultipleFieldsVisitor.java delete mode 100644 src/main/java/org/elasticsearch/common/lucene/document/BaseFieldVisitor.java delete mode 100644 src/main/java/org/elasticsearch/common/lucene/document/SingleFieldVisitor.java create mode 100644 src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java create mode 100644 src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java create mode 100644 src/main/java/org/elasticsearch/index/fieldvisitor/JustSourceFieldsVisitor.java create mode 100644 src/main/java/org/elasticsearch/index/fieldvisitor/JustUidFieldsVisitor.java create mode 100644 src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java create mode 100644 src/main/java/org/elasticsearch/index/fieldvisitor/UidAndRoutingFieldsVisitor.java create mode 100644 src/main/java/org/elasticsearch/index/fieldvisitor/UidAndSourceFieldsVisitor.java delete mode 100644 src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldVisitor.java delete mode 100644 src/main/java/org/elasticsearch/index/mapper/selector/AllButSourceFieldVisitor.java delete mode 100644 src/main/java/org/elasticsearch/index/mapper/selector/FieldMappersFieldVisitor.java delete mode 100644 src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldVisitor.java delete mode 100644 src/main/java/org/elasticsearch/index/mapper/selector/UidAndSourceFieldVisitor.java delete mode 100644 src/main/java/org/elasticsearch/index/mapper/selector/UidFieldVisitor.java diff --git a/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java b/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java index 97f254cc6a1..5471657aa6b 100644 --- a/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java +++ b/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java @@ -21,7 +21,9 @@ package org.elasticsearch.action.mlt; import org.apache.lucene.document.Field; import org.apache.lucene.index.Term; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; @@ -33,7 +35,10 @@ import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.MutableShardRouting; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardIterator; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.get.GetField; @@ -47,7 +52,6 @@ import org.elasticsearch.transport.*; import java.util.Collections; import java.util.Iterator; -import java.util.List; import java.util.Set; import static com.google.common.collect.Sets.newHashSet; @@ -267,7 +271,7 @@ public class TransportMoreLikeThisAction extends TransportAction fields; + + public CustomFieldsVisitor(Set fields, boolean loadSource) { + this.loadAllFields = false; + this.loadSource = loadSource; + this.fields = fields; + } + + public CustomFieldsVisitor(boolean loadAllFields, boolean loadSource) { + this.loadAllFields = loadAllFields; + this.loadSource = loadSource; + this.fields = null; + } + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (loadAllFields) { + return Status.YES; + } + if (loadSource && SourceFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } + if (UidFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } + + return fields.contains(fieldInfo.name) ? Status.YES : Status.NO; + } +} diff --git a/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java new file mode 100644 index 00000000000..34b858fbf3c --- /dev/null +++ b/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -0,0 +1,151 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fieldvisitor; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.StoredFieldVisitor; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.internal.SourceFieldMapper; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.Maps.newHashMap; + +/** + */ +public abstract class FieldsVisitor extends StoredFieldVisitor { + + protected BytesReference source; + protected Uid uid; + protected Map> fieldsValues; + + public void postProcess(MapperService mapperService) { + if (fieldsValues == null || fieldsValues.isEmpty()) { + return; + } + if (uid != null) { + DocumentMapper documentMapper = mapperService.documentMapper(uid.type()); + if (documentMapper != null) { + // we can derive the exact type for the mapping + postProcess(documentMapper); + return; + } + } + // can't derive exact mapping type + for (Map.Entry> entry : fieldsValues.entrySet()) { + FieldMappers fieldMappers = mapperService.indexName(entry.getKey()); + if (fieldMappers == null) { + continue; + } + List fieldValues = entry.getValue(); + for (int i = 0; i < fieldValues.size(); i++) { + fieldValues.set(i, fieldMappers.mapper().valueForSearch(fieldValues.get(i))); + } + } + } + + public void postProcess(DocumentMapper documentMapper) { + for (Map.Entry> entry : fieldsValues.entrySet()) { + FieldMapper fieldMapper = documentMapper.mappers().indexName(entry.getKey()).mapper(); + if (fieldMapper == null) { + continue; + } + List fieldValues = entry.getValue(); + for (int i = 0; i < fieldValues.size(); i++) { + fieldValues.set(i, fieldMapper.valueForSearch(fieldValues.get(i))); + } + } + } + + @Override + public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { + if (SourceFieldMapper.NAME.equals(fieldInfo.name)) { + source = new BytesArray(value); + } else { + addValue(fieldInfo.name, new BytesArray(value)); + } + } + + @Override + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + if (UidFieldMapper.NAME.equals(fieldInfo.name)) { + uid = Uid.createUid(value); + } else { + addValue(fieldInfo.name, value); + } + } + + @Override + public void intField(FieldInfo fieldInfo, int value) throws IOException { + addValue(fieldInfo.name, value); + } + + @Override + public void longField(FieldInfo fieldInfo, long value) throws IOException { + addValue(fieldInfo.name, value); + } + + @Override + public void floatField(FieldInfo fieldInfo, float value) throws IOException { + addValue(fieldInfo.name, value); + } + + @Override + public void doubleField(FieldInfo fieldInfo, double value) throws IOException { + addValue(fieldInfo.name, value); + } + + public BytesReference source() { + return source; + } + + public Uid uid() { + return uid; + } + + public Map> fields() { + return fieldsValues; + } + + public void reset() { + if (fieldsValues != null) fieldsValues.clear(); + source = null; + uid = null; + } + + private void addValue(String name, Object value) { + if (fieldsValues == null) { + fieldsValues = newHashMap(); + } + + List values = fieldsValues.get(name); + if (values == null) { + values = new ArrayList(2); + fieldsValues.put(name, values); + } + values.add(value); + } +} diff --git a/src/main/java/org/elasticsearch/index/fieldvisitor/JustSourceFieldsVisitor.java b/src/main/java/org/elasticsearch/index/fieldvisitor/JustSourceFieldsVisitor.java new file mode 100644 index 00000000000..8f1f2013de6 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/fieldvisitor/JustSourceFieldsVisitor.java @@ -0,0 +1,38 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fieldvisitor; + +import org.apache.lucene.index.FieldInfo; +import org.elasticsearch.index.mapper.internal.SourceFieldMapper; + +import java.io.IOException; + +/** + */ +public class JustSourceFieldsVisitor extends FieldsVisitor { + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (SourceFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } + return source != null ? Status.STOP : Status.NO; + } +} diff --git a/src/main/java/org/elasticsearch/index/fieldvisitor/JustUidFieldsVisitor.java b/src/main/java/org/elasticsearch/index/fieldvisitor/JustUidFieldsVisitor.java new file mode 100644 index 00000000000..d28a4db7eb9 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/fieldvisitor/JustUidFieldsVisitor.java @@ -0,0 +1,38 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fieldvisitor; + +import org.apache.lucene.index.FieldInfo; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; + +import java.io.IOException; + +/** + */ +public class JustUidFieldsVisitor extends FieldsVisitor { + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (UidFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } + return uid != null ? Status.STOP : Status.NO; + } +} diff --git a/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java b/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java new file mode 100644 index 00000000000..a204a421390 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java @@ -0,0 +1,65 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fieldvisitor; + +import org.apache.lucene.index.FieldInfo; +import org.elasticsearch.index.mapper.FieldMapper; + +import java.io.IOException; +import java.util.List; + +/** + */ +public class SingleFieldsVisitor extends FieldsVisitor { + + private String field; + + public SingleFieldsVisitor(String field) { + this.field = field; + } + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + // TODO we can potentially skip if we processed a field, the question is if it works for multi valued fields + if (fieldInfo.name.equals(field)) { + return Status.YES; + } else { + return Status.NO; + } + } + + public void reset(String field) { + this.field = field; + super.reset(); + } + + public void postProcess(FieldMapper mapper) { + if (fieldsValues == null) { + return; + } + List fieldValues = fieldsValues.get(mapper.names().indexName()); + if (fieldValues == null) { + return; + } + for (int i = 0; i < fieldValues.size(); i++) { + fieldValues.set(i, mapper.valueForSearch(fieldValues.get(i))); + } + } +} diff --git a/src/main/java/org/elasticsearch/index/fieldvisitor/UidAndRoutingFieldsVisitor.java b/src/main/java/org/elasticsearch/index/fieldvisitor/UidAndRoutingFieldsVisitor.java new file mode 100644 index 00000000000..6d244867351 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/fieldvisitor/UidAndRoutingFieldsVisitor.java @@ -0,0 +1,57 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fieldvisitor; + +import org.apache.lucene.index.FieldInfo; +import org.elasticsearch.index.mapper.internal.RoutingFieldMapper; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; + +import java.io.IOException; + +/** + */ +public class UidAndRoutingFieldsVisitor extends FieldsVisitor { + + private String routing; + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } else if (UidFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } + + return uid != null && routing != null ? Status.STOP : Status.NO; + } + + @Override + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) { + routing = value; + } else { + super.stringField(fieldInfo, value); + } + } + + public String routing() { + return routing; + } +} diff --git a/src/main/java/org/elasticsearch/index/fieldvisitor/UidAndSourceFieldsVisitor.java b/src/main/java/org/elasticsearch/index/fieldvisitor/UidAndSourceFieldsVisitor.java new file mode 100644 index 00000000000..09ad9bcc3c2 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/fieldvisitor/UidAndSourceFieldsVisitor.java @@ -0,0 +1,42 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fieldvisitor; + +import org.apache.lucene.index.FieldInfo; +import org.elasticsearch.index.mapper.internal.SourceFieldMapper; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; + +import java.io.IOException; + +/** + */ +public class UidAndSourceFieldsVisitor extends FieldsVisitor { + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (SourceFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } else if (UidFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } + + return uid != null && source != null ? Status.STOP : Status.NO; + } +} diff --git a/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 5e732dc1dc5..1147bcd6f65 100644 --- a/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -19,23 +19,22 @@ package org.elasticsearch.index.get; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexableField; +import com.google.common.collect.Sets; import org.apache.lucene.index.Term; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.lucene.document.BaseFieldVisitor; import org.elasticsearch.common.lucene.uid.UidField; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.cache.IndexCache; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor; +import org.elasticsearch.index.fieldvisitor.FieldsVisitor; +import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor; import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.internal.*; -import org.elasticsearch.index.mapper.selector.FieldMappersFieldVisitor; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; @@ -48,6 +47,8 @@ import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -274,50 +275,23 @@ public class ShardGetService extends AbstractIndexShardComponent { private GetResult innerGetLoadFromStoredFields(String type, String id, String[] gFields, Engine.GetResult get, DocumentMapper docMapper) { Map fields = null; - byte[] source = null; + BytesReference source = null; UidField.DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); - // LUCENE 4 UPGRADE: optimize when only a single field needs to be loaded - BaseFieldVisitor fieldVisitor = buildFieldSelectors(docMapper, gFields); + FieldsVisitor fieldVisitor = buildFieldsVisitors(gFields); if (fieldVisitor != null) { - Document doc; try { docIdAndVersion.reader.reader().document(docIdAndVersion.docId, fieldVisitor); - doc = fieldVisitor.createDocument(); } catch (IOException e) { throw new ElasticSearchException("Failed to get type [" + type + "] and id [" + id + "]", e); } - source = extractSource(doc, docMapper); + source = fieldVisitor.source(); - for (Object oField : doc.getFields()) { - Field field = (Field) oField; - String name = field.name(); - Object value = null; - FieldMappers fieldMappers = docMapper.mappers().indexName(field.name()); - if (fieldMappers != null) { - FieldMapper mapper = fieldMappers.mapper(); - if (mapper != null) { - name = mapper.names().fullName(); - value = mapper.valueForSearch(field); - } + if (fieldVisitor.fields() != null) { + fieldVisitor.postProcess(docMapper); + fields = new HashMap(fieldVisitor.fields().size()); + for (Map.Entry> entry : fieldVisitor.fields().entrySet()) { + fields.put(entry.getKey(), new GetField(entry.getKey(), entry.getValue())); } - if (value == null) { - if (field.binaryValue() != null) { - value = new BytesArray(field.binaryValue()); - } else { - value = field.stringValue(); - } - } - - if (fields == null) { - fields = newHashMapWithExpectedSize(2); - } - - GetField getField = fields.get(name); - if (getField == null) { - getField = new GetField(name, new ArrayList(2)); - fields.put(name, getField); - } - getField.values().add(value); } } @@ -368,12 +342,12 @@ public class ShardGetService extends AbstractIndexShardComponent { } } - return new GetResult(shardId.index().name(), type, id, get.version(), get.exists(), source == null ? null : new BytesArray(source), fields); + return new GetResult(shardId.index().name(), type, id, get.version(), get.exists(), source, fields); } - private static BaseFieldVisitor buildFieldSelectors(DocumentMapper docMapper, String... fields) { + private static FieldsVisitor buildFieldsVisitors(String... fields) { if (fields == null) { - return docMapper.sourceMapper().fieldSelector(); + return new JustSourceFieldsVisitor(); } // don't load anything @@ -381,28 +355,6 @@ public class ShardGetService extends AbstractIndexShardComponent { return null; } - FieldMappersFieldVisitor fieldVisitor = null; - for (String fieldName : fields) { - FieldMappers x = docMapper.mappers().smartName(fieldName); - if (x != null && x.mapper().stored()) { - if (fieldVisitor == null) { - fieldVisitor = new FieldMappersFieldVisitor(); - } - fieldVisitor.add(x); - } - } - - return fieldVisitor; - } - - private static byte[] extractSource(Document doc, DocumentMapper documentMapper) { - byte[] source = null; - IndexableField sourceField = doc.getField(documentMapper.sourceMapper().names().indexName()); - if (sourceField != null) { - // LUCENE 4 UPGRADE: Field instead of IndexableField? - source = documentMapper.sourceMapper().nativeValue((Field) sourceField); - doc.removeField(documentMapper.sourceMapper().names().indexName()); - } - return source; + return new CustomFieldsVisitor(Sets.newHashSet(fields), false); } } diff --git a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 571525a4970..c20da27c757 100644 --- a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Field; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.search.Filter; @@ -166,19 +165,19 @@ public interface FieldMapper { /** * Returns the value that will be used as a result for search. Can be only of specific types... . */ - Object valueForSearch(Field field); + Object valueForSearch(Object value); /** * Returns the actual value of the field. */ - T value(Field field); + T value(Object value); T valueFromString(String value); /** * Returns the actual value of the field as string. */ - String valueAsString(Field field); + String valueAsString(Object value); /** * Returns the indexed value. diff --git a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java index 1e709d61004..daf1378440c 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java @@ -415,8 +415,8 @@ public abstract class AbstractFieldMapper implements FieldMapper, Mapper { } @Override - public Object valueForSearch(Field field) { - return valueAsString(field); + public Object valueForSearch(Object value) { + return valueAsString(value); } @Override @@ -424,6 +424,11 @@ public abstract class AbstractFieldMapper implements FieldMapper, Mapper { return value; } + @Override + public String valueAsString(Object value) { + return String.valueOf(value); + } + @Override public Query queryStringTermQuery(Term term) { return null; diff --git a/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java index b8bb849a2f2..860979f6a8a 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java @@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper.core; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticSearchParseException; import org.elasticsearch.common.Base64; import org.elasticsearch.common.Strings; @@ -126,18 +125,17 @@ public class BinaryFieldMapper extends AbstractFieldMapper { } @Override - public Object valueForSearch(Field field) { - return value(field); + public Object valueForSearch(Object value) { + return value(value); } @Override - public byte[] value(Field field) { - BytesRef value = field.binaryValue(); + public byte[] value(Object value) { if (value == null) { return null; } try { - return CompressorFactory.uncompressIfNeeded(new BytesArray(value)).toBytes(); + return CompressorFactory.uncompressIfNeeded(new BytesArray((byte[]) value)).toBytes(); } catch (IOException e) { throw new ElasticSearchParseException("failed to decompress source", e); } @@ -154,7 +152,7 @@ public class BinaryFieldMapper extends AbstractFieldMapper { } @Override - public String valueAsString(Field field) { + public String valueAsString(Object value) { return null; } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java index 68438e7f83b..10a511cc0a8 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java @@ -154,8 +154,8 @@ public class BooleanFieldMapper extends AbstractFieldMapper { } @Override - public Boolean value(Field field) { - return field.stringValue().charAt(0) == 'T' ? Boolean.TRUE : Boolean.FALSE; + public Boolean value(Object value) { + return valueFromString((String) value); } @Override @@ -164,8 +164,8 @@ public class BooleanFieldMapper extends AbstractFieldMapper { } @Override - public String valueAsString(Field field) { - return field.stringValue().charAt(0) == 'T' ? "true" : "false"; + public String valueAsString(Object value) { + return ((String) value).charAt(0) == 'T' ? "true" : "false"; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index a5b178e44da..c37f16b42ed 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -32,6 +32,7 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -128,12 +129,15 @@ public class ByteFieldMapper extends NumberFieldMapper { } @Override - public Byte value(Field field) { - BytesRef value = field.binaryValue(); + public Byte value(Object value) { if (value == null) { return null; } - return value.bytes[value.offset]; + if (value instanceof Number) { + return ((Number) value).byteValue(); + } + BytesReference bytesReference = (BytesReference) value; + return bytesReference.get(bytesReference.arrayOffset()); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 01defbc560c..bfe3bed1ef6 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.joda.DateMathParser; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; @@ -184,12 +185,14 @@ public class DateFieldMapper extends NumberFieldMapper { } @Override - public Long value(Field field) { - BytesRef value = field.binaryValue(); + public Long value(Object value) { if (value == null) { return null; } - return Numbers.bytesToLong(value.bytes); + if (value instanceof Number) { + return ((Number) value).longValue(); + } + return Numbers.bytesToLong(((BytesReference) value).array()); } @Override @@ -199,19 +202,21 @@ public class DateFieldMapper extends NumberFieldMapper { /** * Dates should return as a string. + * + * @param value */ @Override - public Object valueForSearch(Field field) { - return valueAsString(field); + public Object valueForSearch(Object value) { + return valueAsString(value); } @Override - public String valueAsString(Field field) { - Long value = value(field); - if (value == null) { + public String valueAsString(Object value) { + Long val = value(value); + if (val == null) { return null; } - return dateTimeFormatter.printer().print(value); + return dateTimeFormatter.printer().print(val); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index c3069c47518..ff9ecdad472 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -32,6 +32,7 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -131,12 +132,14 @@ public class DoubleFieldMapper extends NumberFieldMapper { } @Override - public Double value(Field field) { - BytesRef value = field.binaryValue(); + public Double value(Object value) { if (value == null) { return null; } - return Numbers.bytesToDouble(value.bytes); + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + return Numbers.bytesToDouble(((BytesReference) value).array()); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index dbfedcfb5ba..169b7b5b482 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -129,12 +130,14 @@ public class FloatFieldMapper extends NumberFieldMapper { } @Override - public Float value(Field field) { - BytesRef value = field.binaryValue(); + public Float value(Object value) { if (value == null) { return null; } - return Numbers.bytesToFloat(value.bytes); + if (value instanceof Number) { + return ((Number) value).floatValue(); + } + return Numbers.bytesToFloat(((BytesReference) value).array()); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index 2038e02aee0..3b20d69bccf 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -131,12 +132,14 @@ public class IntegerFieldMapper extends NumberFieldMapper { } @Override - public Integer value(Field field) { - BytesRef value = field.binaryValue(); + public Integer value(Object value) { if (value == null) { return null; } - return Numbers.bytesToInt(value.bytes); + if (value instanceof Number) { + return ((Number) value).intValue(); + } + return Numbers.bytesToInt(((BytesReference) value).array()); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index 18945a0db73..5cab9b6d1f2 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -131,12 +132,14 @@ public class LongFieldMapper extends NumberFieldMapper { } @Override - public Long value(Field field) { - BytesRef value = field.binaryValue(); + public Long value(Object value) { if (value == null) { return null; } - return Numbers.bytesToLong(value.bytes); + if (value instanceof Number) { + return ((Number) value).longValue(); + } + return Numbers.bytesToLong(((BytesReference) value).array()); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java index 06b26ac0021..c104b3ad178 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java @@ -247,15 +247,17 @@ public abstract class NumberFieldMapper extends AbstractFieldM /** * Override the default behavior (to return the string, and return the actual Number instance). + * + * @param value */ @Override - public Object valueForSearch(Field field) { - return value(field); + public Object valueForSearch(Object value) { + return value(value); } @Override - public String valueAsString(Field field) { - Number num = value(field); + public String valueAsString(Object value) { + Number num = value(value); return num == null ? null : num.toString(); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 45f520572f8..c0d456d674a 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -131,12 +132,14 @@ public class ShortFieldMapper extends NumberFieldMapper { } @Override - public Short value(Field field) { - BytesRef value = field.binaryValue(); + public Short value(Object value) { if (value == null) { return null; } - return Numbers.bytesToShort(value.bytes); + if (value instanceof Number) { + return ((Number) value).shortValue(); + } + return Numbers.bytesToShort(((BytesReference) value).array()); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java index 51cbc1525ad..d96d7f84b48 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java @@ -217,8 +217,8 @@ public class StringFieldMapper extends AbstractFieldMapper implements Al } @Override - public String value(Field field) { - return field.stringValue(); + public String value(Object value) { + return String.valueOf(value); } @Override @@ -226,11 +226,6 @@ public class StringFieldMapper extends AbstractFieldMapper implements Al return value; } - @Override - public String valueAsString(Field field) { - return value(field); - } - @Override public String indexedValue(String value) { return value; diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java index 0dc5b4c836f..977f5d61e3f 100644 --- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java @@ -176,7 +176,7 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper { } @Override - public String value(Field field) { + public String value(Object value) { throw new UnsupportedOperationException("GeoShape fields cannot be converted to String values"); } @@ -186,7 +186,7 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper { } @Override - public String valueAsString(Field field) { + public String valueAsString(Object value) { throw new UnsupportedOperationException("GeoShape fields cannot be converted to String values"); } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java index 5a3b3c79590..1d1d8889c4b 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java @@ -221,7 +221,7 @@ public class AllFieldMapper extends AbstractFieldMapper implements Interna } @Override - public Void value(Field field) { + public Void value(Object value) { return null; } @@ -231,12 +231,12 @@ public class AllFieldMapper extends AbstractFieldMapper implements Interna } @Override - public String valueAsString(Field field) { + public String valueAsString(Object value) { return null; } @Override - public Object valueForSearch(Field field) { + public Object valueForSearch(Object value) { return null; } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java index 4c91cbaba6f..e723d510429 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java @@ -132,12 +132,11 @@ public class BoostFieldMapper extends NumberFieldMapper implements Intern } @Override - public Float value(Field field) { - BytesRef value = field.binaryValue(); + public Float value(Object value) { if (value == null) { return null; } - return Numbers.bytesToFloat(value.bytes); + return Numbers.bytesToFloat((byte[]) value); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java index 3330cf98de2..424590de745 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java @@ -139,8 +139,8 @@ public class IdFieldMapper extends AbstractFieldMapper implements Intern } @Override - public String value(Field field) { - return field.stringValue(); + public String value(Object value) { + return String.valueOf(value); } @Override @@ -148,11 +148,6 @@ public class IdFieldMapper extends AbstractFieldMapper implements Intern return value; } - @Override - public String valueAsString(Field field) { - return value(field); - } - @Override public String indexedValue(String value) { return value; diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java index 25dbc37f56a..c39a2e46207 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java @@ -128,8 +128,8 @@ public class IndexFieldMapper extends AbstractFieldMapper implements Int } @Override - public String value(Field field) { - return field.stringValue(); + public String value(Object value) { + return String.valueOf(value); } @Override @@ -137,11 +137,6 @@ public class IndexFieldMapper extends AbstractFieldMapper implements Int return value; } - @Override - public String valueAsString(Field field) { - return value(field); - } - @Override public String indexedValue(String value) { return value; diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java index 61372296670..431c2592578 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java @@ -169,8 +169,8 @@ public class ParentFieldMapper extends AbstractFieldMapper implements Inter } @Override - public Uid value(Field field) { - return Uid.createUid(field.stringValue()); + public Uid value(Object value) { + return Uid.createUid(String.valueOf(value)); } @Override @@ -179,13 +179,8 @@ public class ParentFieldMapper extends AbstractFieldMapper implements Inter } @Override - public String valueAsString(Field field) { - return field.stringValue(); - } - - @Override - public Object valueForSearch(Field field) { - String fieldValue = field.stringValue(); + public Object valueForSearch(Object value) { + String fieldValue = String.valueOf(value); if (fieldValue == null) { return null; } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java index de64a9842ae..5dee49e3e72 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java @@ -142,8 +142,8 @@ public class RoutingFieldMapper extends AbstractFieldMapper implements I } @Override - public String value(Field field) { - return field.stringValue(); + public String value(Object value) { + return String.valueOf(value); } @Override @@ -151,11 +151,6 @@ public class RoutingFieldMapper extends AbstractFieldMapper implements I return value; } - @Override - public String valueAsString(Field field) { - return value(field); - } - @Override public String indexedValue(String value) { return value; diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java index 0c460622678..8b6192cf5e9 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java @@ -27,7 +27,6 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.elasticsearch.ElasticSearchParseException; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedStreamInput; @@ -36,7 +35,6 @@ import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.io.stream.CachedStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.lucene.document.BaseFieldVisitor; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -214,10 +212,6 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In return this.enabled; } - public BaseFieldVisitor fieldSelector() { - return new SourceFieldVisitor(); - } - @Override public void preParse(ParseContext context) throws IOException { super.parse(context); @@ -357,13 +351,13 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In } @Override - public byte[] value(Field field) { - byte[] value = field.binaryValue().bytes; - if (value == null) { - return value; + public byte[] value(Object value) { + BytesReference val = (BytesReference) value; + if (val == null) { + return null; } try { - return CompressorFactory.uncompressIfNeeded(new BytesArray(value)).toBytes(); + return CompressorFactory.uncompressIfNeeded(val).toBytes(); } catch (IOException e) { throw new ElasticSearchParseException("failed to decompress source", e); } @@ -375,7 +369,7 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In } @Override - public String valueAsString(Field field) { + public String valueAsString(Object value) { throw new UnsupportedOperationException(); } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldVisitor.java b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldVisitor.java deleted file mode 100644 index 5c9a2d0433d..00000000000 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldVisitor.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.internal; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.lucene.document.BaseFieldVisitor; - -import java.io.IOException; - -/** - * An optimized field selector that loads just the _source - */ -public class SourceFieldVisitor extends BaseFieldVisitor { - - private BytesRef source; - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (SourceFieldMapper.NAME.equals(fieldInfo.name)) { - return Status.YES; - } - return source != null ? Status.STOP : Status.NO; - } - - @Override - public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { - source = new BytesRef(value); - } - - @Override - public Document createDocument() { - Document document = new Document(); - if (source != null) { - document.add(new StoredField(SourceFieldMapper.NAME, source)); - } - return document; - } - - @Override - public void reset() { - source = null; - } - - public BytesRef source() { - return source; - } - - @Override - public String toString() { - return "source"; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index a7dda9109b2..7bda668d0e1 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -134,7 +134,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R // Overrides valueForSearch to display live value of remaining ttl @Override - public Object valueForSearch(Field field) { + public Object valueForSearch(Object value) { long now; SearchContext searchContext = SearchContext.current(); if (searchContext != null) { @@ -142,8 +142,8 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R } else { now = System.currentTimeMillis(); } - long value = value(field); - return value - now; + long val = value(value); + return val - now; } // Other implementation for realtime get display diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index f8a306ee9f4..ce5aef40775 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -157,19 +157,21 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap /** * Override the default behavior to return a timestamp + * + * @param value */ @Override - public Object valueForSearch(Field field) { - return value(field); + public Object valueForSearch(Object value) { + return value(value); } @Override - public String valueAsString(Field field) { - Long value = value(field); - if (value == null) { + public String valueAsString(Object value) { + Long val = value(value); + if (val == null) { return null; } - return value.toString(); + return val.toString(); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java index 68de5e92c02..ff92533bb73 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java @@ -112,8 +112,8 @@ public class TypeFieldMapper extends AbstractFieldMapper implements Inte } @Override - public String value(Field field) { - return field.stringValue(); + public String value(Object value) { + return String.valueOf(value); } @Override @@ -121,11 +121,6 @@ public class TypeFieldMapper extends AbstractFieldMapper implements Inte return value; } - @Override - public String valueAsString(Field field) { - return value(field); - } - @Override public String indexedValue(String value) { return value; diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java index 1fd62232e3e..3b11ed5f287 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java @@ -175,8 +175,8 @@ public class UidFieldMapper extends AbstractFieldMapper implements Internal } @Override - public Uid value(Field field) { - return Uid.createUid(field.stringValue()); + public Uid value(Object value) { + return Uid.createUid(String.valueOf(value)); } @Override @@ -184,11 +184,6 @@ public class UidFieldMapper extends AbstractFieldMapper implements Internal return Uid.createUid(value); } - @Override - public String valueAsString(Field field) { - return field.stringValue(); - } - @Override public String indexedValue(String value) { return value; diff --git a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index 6af808d3bd5..651036cee7b 100644 --- a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -154,12 +154,11 @@ public class IpFieldMapper extends NumberFieldMapper { } @Override - public Long value(Field field) { - byte[] value = field.binaryValue().bytes; + public Long value(Object value) { if (value == null) { return null; } - return Numbers.bytesToLong(value); + return Numbers.bytesToLong((byte[]) value); } @Override @@ -169,19 +168,21 @@ public class IpFieldMapper extends NumberFieldMapper { /** * IPs should return as a string. + * + * @param value */ @Override - public Object valueForSearch(Field field) { - return valueAsString(field); + public Object valueForSearch(Object value) { + return valueAsString(value); } @Override - public String valueAsString(Field field) { - Long value = value(field); - if (value == null) { + public String valueAsString(Object value) { + Long val = value(value); + if (val == null) { return null; } - return longToIp(value); + return longToIp(val); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/selector/AllButSourceFieldVisitor.java b/src/main/java/org/elasticsearch/index/mapper/selector/AllButSourceFieldVisitor.java deleted file mode 100644 index ef89bf897a0..00000000000 --- a/src/main/java/org/elasticsearch/index/mapper/selector/AllButSourceFieldVisitor.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.selector; - -import org.apache.lucene.index.FieldInfo; -import org.elasticsearch.common.lucene.document.AbstractMultipleFieldsVisitor; -import org.elasticsearch.index.mapper.internal.SourceFieldMapper; - -import java.io.IOException; - -/** - * A field selector that loads all fields except the source field. - */ -public class AllButSourceFieldVisitor extends AbstractMultipleFieldsVisitor { - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (SourceFieldMapper.NAME.equals(fieldInfo.name)) { - return Status.NO; - } - return Status.YES; - } - - @Override - public String toString() { - return "all_but_source"; - } -} diff --git a/src/main/java/org/elasticsearch/index/mapper/selector/FieldMappersFieldVisitor.java b/src/main/java/org/elasticsearch/index/mapper/selector/FieldMappersFieldVisitor.java deleted file mode 100644 index 87ffa34fbd5..00000000000 --- a/src/main/java/org/elasticsearch/index/mapper/selector/FieldMappersFieldVisitor.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.selector; - -import org.apache.lucene.index.FieldInfo; -import org.elasticsearch.common.lucene.document.AbstractMultipleFieldsVisitor; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.FieldMappers; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Set; - -/** - * - */ -public class FieldMappersFieldVisitor extends AbstractMultipleFieldsVisitor { - - protected final Set fieldsToAdd = new HashSet(); - - public void add(String fieldName) { - fieldsToAdd.add(fieldName); - } - - public void add(FieldMappers fieldMappers) { - for (FieldMapper fieldMapper : fieldMappers) { - fieldsToAdd.add(fieldMapper.names().indexName()); - } - } - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - return fieldsToAdd.contains(fieldInfo.name) ? Status.YES : Status.NO; - } - - @Override - public String toString() { - return "fields(" + fieldsToAdd + ")"; - } -} diff --git a/src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldVisitor.java b/src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldVisitor.java deleted file mode 100644 index 0cf896a1d4d..00000000000 --- a/src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldVisitor.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.selector; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.FieldInfo; -import org.elasticsearch.common.lucene.document.BaseFieldVisitor; -import org.elasticsearch.index.mapper.internal.RoutingFieldMapper; -import org.elasticsearch.index.mapper.internal.SourceFieldMapper; -import org.elasticsearch.index.mapper.internal.UidFieldMapper; - -import java.io.IOException; - -/** - * An optimized field selector that loads just the uid and the routing. - */ -public class UidAndRoutingFieldVisitor extends BaseFieldVisitor { - - private String uid; - private String routing; - - @Override - public Document createDocument() { - Document document = new Document(); - if (uid != null) { - document.add(new StoredField(UidFieldMapper.NAME, uid)); - } - if (routing != null) { - document.add(new StoredField(SourceFieldMapper.NAME, routing)); - } - return document; - } - - @Override - public void reset() { - uid = null; - routing = null; - } - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) { - return Status.YES; - } else if (UidFieldMapper.NAME.equals(fieldInfo.name)) { - return Status.YES; - } - - return uid != null && routing != null ? Status.STOP : Status.NO; - } - - @Override - public void stringField(FieldInfo fieldInfo, String value) throws IOException { - if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) { - routing = value; - } else if (UidFieldMapper.NAME.equals(fieldInfo.name)) { - uid = value; - } - } - - public String uid() { - return uid; - } - - public String routing() { - return routing; - } - - @Override - public String toString() { - return "uid_and_routing"; - } -} diff --git a/src/main/java/org/elasticsearch/index/mapper/selector/UidAndSourceFieldVisitor.java b/src/main/java/org/elasticsearch/index/mapper/selector/UidAndSourceFieldVisitor.java deleted file mode 100644 index b9cf0a01327..00000000000 --- a/src/main/java/org/elasticsearch/index/mapper/selector/UidAndSourceFieldVisitor.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.selector; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.lucene.document.BaseFieldVisitor; -import org.elasticsearch.index.mapper.internal.SourceFieldMapper; -import org.elasticsearch.index.mapper.internal.UidFieldMapper; - -import java.io.IOException; - -/** - * An optimized field selector that loads just the uid and the source. - */ -public class UidAndSourceFieldVisitor extends BaseFieldVisitor { - - private String uid; - private BytesRef source; - - @Override - public Document createDocument() { - Document document = new Document(); - if (uid != null) { - document.add(new StoredField(UidFieldMapper.NAME, uid)); - } - if (source != null) { - document.add(new StoredField(SourceFieldMapper.NAME, source)); - } - return document; - } - - @Override - public void reset() { - source = null; - uid = null; - } - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (SourceFieldMapper.NAME.equals(fieldInfo.name)) { - return Status.YES; - } else if (UidFieldMapper.NAME.equals(fieldInfo.name)) { - return Status.YES; - } - - return uid != null && source != null ? Status.STOP : Status.NO; - } - - @Override - public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { - source = new BytesRef(value); - } - - @Override - public void stringField(FieldInfo fieldInfo, String value) throws IOException { - uid = value; - } - - @Override - public String toString() { - return "uid_and_source"; - } -} diff --git a/src/main/java/org/elasticsearch/index/mapper/selector/UidFieldVisitor.java b/src/main/java/org/elasticsearch/index/mapper/selector/UidFieldVisitor.java deleted file mode 100644 index 2caf597c788..00000000000 --- a/src/main/java/org/elasticsearch/index/mapper/selector/UidFieldVisitor.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.selector; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.FieldInfo; -import org.elasticsearch.common.lucene.document.BaseFieldVisitor; -import org.elasticsearch.index.mapper.internal.UidFieldMapper; - -import java.io.IOException; - -/** - * An optimized field selector that loads just the uid. - */ -public class UidFieldVisitor extends BaseFieldVisitor { - - private String uid; - - public UidFieldVisitor() { - } - - @Override - public void stringField(FieldInfo fieldInfo, String value) throws IOException { - uid = value; - } - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (UidFieldMapper.NAME.equals(fieldInfo.name)) { - return Status.YES; - } - return uid != null ? Status.STOP : Status.NO; - } - - @Override - public Document createDocument() { - Document document = new Document(); - if (uid != null) { - document.add(new StoredField(UidFieldMapper.NAME, uid)); - } - return document; - } - - @Override - public void reset() { - uid = null; - } - - public String uid() { - return uid; - } - - @Override - public String toString() { - return "uid"; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/percolator/PercolatorService.java b/src/main/java/org/elasticsearch/index/percolator/PercolatorService.java index f53dbe25786..a2a0fe28c3c 100644 --- a/src/main/java/org/elasticsearch/index/percolator/PercolatorService.java +++ b/src/main/java/org/elasticsearch/index/percolator/PercolatorService.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.percolator; import com.google.common.collect.Maps; -import org.apache.lucene.document.Document; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.Term; @@ -28,8 +27,6 @@ import org.apache.lucene.search.Collector; import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.TermFilter; import org.elasticsearch.common.lucene.search.XConstantScoreQuery; @@ -37,12 +34,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.fieldvisitor.UidAndSourceFieldsVisitor; import org.elasticsearch.index.indexing.IndexingOperationListener; -import org.elasticsearch.index.mapper.Uid; -import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; -import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.index.mapper.selector.UidAndSourceFieldVisitor; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.IndexShardState; @@ -178,13 +172,11 @@ public class PercolatorService extends AbstractIndexComponent { @Override public void collect(int doc) throws IOException { // the _source is the query - UidAndSourceFieldVisitor fieldVisitor = new UidAndSourceFieldVisitor(); - reader.document(doc, fieldVisitor); - Document document = fieldVisitor.createDocument(); - String id = Uid.createUid(document.get(UidFieldMapper.NAME)).id(); + UidAndSourceFieldsVisitor fieldsVisitor = new UidAndSourceFieldsVisitor(); + reader.document(doc, fieldsVisitor); + String id = fieldsVisitor.uid().id(); try { - BytesRef sourceVal = document.getBinaryValue(SourceFieldMapper.NAME); - queries.put(id, percolator.parseQuery(id, new BytesArray(sourceVal.bytes, sourceVal.offset, sourceVal.length))); + queries.put(id, percolator.parseQuery(id, fieldsVisitor.source())); } catch (Exception e) { logger.warn("failed to add query [{}]", e, id); } diff --git a/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java b/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java index 9c4298b36c5..fd137645612 100644 --- a/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java +++ b/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java @@ -41,12 +41,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.fieldvisitor.UidAndRoutingFieldsVisitor; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMappers; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.internal.TTLFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.index.mapper.selector.UidAndRoutingFieldVisitor; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.service.IndexShard; @@ -188,7 +189,7 @@ public class IndicesTTLService extends AbstractLifecycleComponent docsToPurge = expiredDocsCollector.getDocsToPurge(); BulkRequestBuilder bulkRequest = client.prepareBulk(); @@ -220,10 +221,12 @@ public class IndicesTTLService extends AbstractLifecycleComponent docsToPurge = new ArrayList(); - public ExpiredDocsCollector() { + public ExpiredDocsCollector(String index) { + mapperService = indicesService.indexService(index).mapperService(); } public void setScorer(Scorer scorer) { @@ -235,11 +238,11 @@ public class IndicesTTLService extends AbstractLifecycleComponent extractFieldNames = null; boolean sourceRequested = false; if (!context.hasFieldNames()) { if (context.hasPartialFields()) { // partial fields need the source, so fetch it, but don't return it - fieldVisitor = new UidAndSourceFieldVisitor(); - sourceRequested = false; + fieldsVisitor = new UidAndSourceFieldsVisitor(); } else if (context.hasScriptFields()) { // we ask for script fields, and no field names, don't load the source - fieldVisitor = new UidFieldVisitor(); - sourceRequested = false; + fieldsVisitor = new JustUidFieldsVisitor(); } else { - fieldVisitor = new UidAndSourceFieldVisitor(); sourceRequested = true; + fieldsVisitor = new UidAndSourceFieldsVisitor(); } } else if (context.fieldNames().isEmpty()) { - fieldVisitor = new UidFieldVisitor(); - sourceRequested = false; + fieldsVisitor = new JustUidFieldsVisitor(); } else { boolean loadAllStored = false; - FieldMappersFieldVisitor fieldVisitorMapper = null; + Set fieldNames = null; for (String fieldName : context.fieldNames()) { if (fieldName.equals("*")) { loadAllStored = true; @@ -122,101 +107,52 @@ public class FetchPhase implements SearchPhase { } FieldMappers x = context.smartNameFieldMappers(fieldName); if (x != null && x.mapper().stored()) { - if (fieldVisitorMapper == null) { - fieldVisitorMapper = new FieldMappersFieldVisitor(); + if (fieldNames == null) { + fieldNames = new HashSet(); } - fieldVisitorMapper.add(x); + fieldNames.add(x.mapper().names().indexName()); } else { if (extractFieldNames == null) { - extractFieldNames = Lists.newArrayList(); + extractFieldNames = newArrayList(); } extractFieldNames.add(fieldName); } } - if (loadAllStored) { if (sourceRequested || extractFieldNames != null) { - fieldVisitor = null; // load everything, including _source + fieldsVisitor = new CustomFieldsVisitor(true, true); // load everything, including _source } else { - fieldVisitor = new AllButSourceFieldVisitor(); + fieldsVisitor = new CustomFieldsVisitor(true, false); } - } else if (fieldVisitorMapper != null) { - // we are asking specific stored fields, just add the UID and be done - fieldVisitorMapper.add(UidFieldMapper.NAME); - if (extractFieldNames != null || sourceRequested) { - fieldVisitorMapper.add(SourceFieldMapper.NAME); - } - fieldVisitor = fieldVisitorMapper; + } else if (fieldNames != null) { + boolean loadSource = extractFieldNames != null || sourceRequested; + fieldsVisitor = new CustomFieldsVisitor(fieldNames, loadSource); } else if (extractFieldNames != null || sourceRequested) { - fieldVisitor = new UidAndSourceFieldVisitor(); + fieldsVisitor = new UidAndSourceFieldsVisitor(); } else { - fieldVisitor = new UidFieldVisitor(); + fieldsVisitor = new JustUidFieldsVisitor(); } } InternalSearchHit[] hits = new InternalSearchHit[context.docIdsToLoadSize()]; for (int index = 0; index < context.docIdsToLoadSize(); index++) { int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index]; - Document doc = loadDocument(context, fieldVisitor, docId); - Uid uid = extractUid(context, doc, fieldVisitor); - DocumentMapper documentMapper = context.mapperService().documentMapper(uid.type()); + loadStoredFields(context, fieldsVisitor, docId); + fieldsVisitor.postProcess(context.mapperService()); - if (documentMapper == null) { - throw new TypeMissingException(new Index(context.shardTarget().index()), uid.type(), "failed to find type loaded for doc [" + uid.id() + "]"); + Map searchFields = null; + if (fieldsVisitor.fields() != null) { + searchFields = new HashMap(fieldsVisitor.fields().size()); + for (Map.Entry> entry : fieldsVisitor.fields().entrySet()) { + searchFields.put(entry.getKey(), new InternalSearchHitField(entry.getKey(), entry.getValue())); + } } - byte[] source = extractSource(doc, documentMapper); + InternalSearchHit searchHit = new InternalSearchHit(docId, fieldsVisitor.uid().id(), fieldsVisitor.uid().type(), sourceRequested ? fieldsVisitor.source() : null, searchFields); - // get the version - - InternalSearchHit searchHit = new InternalSearchHit(docId, uid.id(), uid.type(), sourceRequested ? source : null, null); hits[index] = searchHit; - for (Object oField : doc.getFields()) { - IndexableField field = (IndexableField) oField; - String name = field.name(); - - // ignore UID, we handled it above - if (name.equals(UidFieldMapper.NAME)) { - continue; - } - - // ignore source, we handled it above - if (name.equals(SourceFieldMapper.NAME)) { - continue; - } - - Object value = null; - FieldMappers fieldMappers = documentMapper.mappers().indexName(field.name()); - if (fieldMappers != null) { - FieldMapper mapper = fieldMappers.mapper(); - if (mapper != null) { - name = mapper.names().fullName(); - // LUCENE 4 UPGRADE: do we really need to use Field instead of IndexableField? - value = mapper.valueForSearch((Field) field); - } - } - if (value == null) { - if (field.binaryValue() != null) { - value = new BytesArray(field.binaryValue()); - } else { - value = field.stringValue(); - } - } - - if (searchHit.fieldsOrNull() == null) { - searchHit.fields(new HashMap(2)); - } - - SearchHitField hitField = searchHit.fields().get(name); - if (hitField == null) { - hitField = new InternalSearchHitField(name, new ArrayList(2)); - searchHit.fields().put(name, hitField); - } - hitField.values().add(value); - } - int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves()); AtomicReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); int subDoc = docId - subReaderContext.docBase; @@ -224,8 +160,8 @@ public class FetchPhase implements SearchPhase { // go over and extract fields that are not mapped / stored context.lookup().setNextReader(subReaderContext); context.lookup().setNextDocId(subDoc); - if (source != null) { - context.lookup().source().setNextSource(new BytesArray(source)); + if (searchHit.source() != null) { + context.lookup().source().setNextSource(new BytesArray(searchHit.source())); } if (extractFieldNames != null) { for (String extractFieldName : extractFieldNames) { @@ -248,7 +184,7 @@ public class FetchPhase implements SearchPhase { for (FetchSubPhase fetchSubPhase : fetchSubPhases) { FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); if (fetchSubPhase.hitExecutionNeeded(context)) { - hitContext.reset(searchHit, subReaderContext, subDoc, context.searcher().getIndexReader(), docId, doc); + hitContext.reset(searchHit, subReaderContext, subDoc, context.searcher().getIndexReader(), docId, fieldsVisitor); fetchSubPhase.hitExecute(context, hitContext); } } @@ -263,36 +199,10 @@ public class FetchPhase implements SearchPhase { context.fetchResult().hits(new InternalSearchHits(hits, context.queryResult().topDocs().totalHits, context.queryResult().topDocs().getMaxScore())); } - private byte[] extractSource(Document doc, DocumentMapper documentMapper) { - IndexableField sourceField = doc.getField(SourceFieldMapper.NAME); - if (sourceField != null) { - //LUCENE 4 UPGRADE: I think all sourceFields are of type Field - return documentMapper.sourceMapper().nativeValue((Field) sourceField); - } - return null; - } - - private Uid extractUid(SearchContext context, Document doc, @Nullable BaseFieldVisitor fieldVisitor) { - String sUid = doc.get(UidFieldMapper.NAME); - if (sUid != null) { - return Uid.createUid(sUid); - } - // no type, nothing to do (should not really happen) - List fieldNames = new ArrayList(); - for (IndexableField field : doc.getFields()) { - fieldNames.add(field.name()); - } - throw new FetchPhaseExecutionException(context, "Failed to load uid from the index, missing internal _uid field, current fields in the doc [" + fieldNames + "], selector [" + fieldVisitor + "]"); - } - - private Document loadDocument(SearchContext context, @Nullable BaseFieldVisitor fieldVisitor, int docId) { + private void loadStoredFields(SearchContext context, FieldsVisitor fieldVisitor, int docId) { + fieldVisitor.reset(); try { - if (fieldVisitor == null) { - return context.searcher().doc(docId); - } - fieldVisitor.reset(); context.searcher().doc(docId, fieldVisitor); - return fieldVisitor.createDocument(); } catch (IOException e) { throw new FetchPhaseExecutionException(context, "Failed to fetch doc id [" + docId + "]", e); } diff --git a/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 92951f90fcf..7141f5aac06 100644 --- a/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -20,11 +20,11 @@ package org.elasticsearch.search.fetch; import com.google.common.collect.Maps; -import org.apache.lucene.document.Document; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.index.fieldvisitor.FieldsVisitor; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.SearchContext; @@ -42,16 +42,16 @@ public interface FetchSubPhase { private int topLevelDocId; private AtomicReaderContext readerContext; private int docId; - private Document doc; + private FieldsVisitor fieldVisitor; private Map cache; - public void reset(InternalSearchHit hit, AtomicReaderContext context, int docId, IndexReader topLevelReader, int topLevelDocId, Document doc) { + public void reset(InternalSearchHit hit, AtomicReaderContext context, int docId, IndexReader topLevelReader, int topLevelDocId, FieldsVisitor fieldVisitor) { this.hit = hit; this.readerContext = context; this.docId = docId; this.topLevelReader = topLevelReader; this.topLevelDocId = topLevelDocId; - this.doc = doc; + this.fieldVisitor = fieldVisitor; } public InternalSearchHit hit() { @@ -78,8 +78,8 @@ public interface FetchSubPhase { return topLevelDocId; } - public Document doc() { - return doc; + public FieldsVisitor fieldVisitor() { + return fieldVisitor; } public Map cache() { diff --git a/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java b/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java index ed208fdf08b..73d5ac411cd 100644 --- a/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java +++ b/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java @@ -60,7 +60,10 @@ public class VersionFetchSubPhase implements FetchSubPhase { // it might make sense to cache the TermDocs on a shared fetch context and just skip here) // it is going to mean we work on the high level multi reader and not the lower level reader as is // the case below... - long version = UidField.loadVersion(hitContext.readerContext(), new Term(UidFieldMapper.NAME, hitContext.doc().get(UidFieldMapper.NAME))); + long version = UidField.loadVersion( + hitContext.readerContext(), + new Term(UidFieldMapper.NAME, hitContext.fieldVisitor().uid().toString()) + ); if (version < 0) { version = -1; } diff --git a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java index 79bfbd4ab42..459a21e6d2a 100644 --- a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java +++ b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java @@ -20,11 +20,10 @@ package org.elasticsearch.search.highlight; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.apache.lucene.search.highlight.*; import org.apache.lucene.search.highlight.Formatter; @@ -34,10 +33,10 @@ import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.FastStringReader; -import org.elasticsearch.common.lucene.document.SingleFieldVisitor; import org.elasticsearch.common.lucene.search.vectorhighlight.SimpleBoundaryScanner2; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.StringText; +import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; @@ -176,15 +175,9 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase { List textsToHighlight; if (mapper.stored()) { try { - SingleFieldVisitor fieldVisitor = new SingleFieldVisitor(mapper.names().indexName()); + CustomFieldsVisitor fieldVisitor = new CustomFieldsVisitor(ImmutableSet.of(mapper.names().indexName()), false); hitContext.reader().document(hitContext.docId(), fieldVisitor); - Document doc = fieldVisitor.createDocument(); - textsToHighlight = new ArrayList(doc.getFields().size()); - for (IndexableField docField : doc.getFields()) { - if (docField.stringValue() != null) { - textsToHighlight.add(docField.stringValue()); - } - } + textsToHighlight = fieldVisitor.fields().get(mapper.names().indexName()); } catch (Exception e) { throw new FetchPhaseExecutionException(context, "Failed to highlight field [" + field.field() + "]", e); } diff --git a/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java index 650240e1895..a353770c309 100644 --- a/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java +++ b/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java @@ -91,11 +91,11 @@ public class InternalSearchHit implements SearchHit { } - public InternalSearchHit(int docId, String id, String type, byte[] source, Map fields) { + public InternalSearchHit(int docId, String id, String type, BytesReference source, Map fields) { this.docId = docId; this.id = id; this.type = type; - this.source = source == null ? null : new BytesArray(source); + this.source = source; this.fields = fields; } diff --git a/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java b/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java index 52b33eaa619..416d61682d4 100644 --- a/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java +++ b/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java @@ -19,13 +19,11 @@ package org.elasticsearch.search.lookup; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexableField; import org.elasticsearch.index.mapper.FieldMapper; import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * @@ -35,7 +33,7 @@ public class FieldLookup { // we can cached mapper completely per name, since its on an index/shard level (the lookup, and it does not change within the scope of a search request) private final FieldMapper mapper; - private Document doc; + private Map> fields; private Object value; @@ -53,12 +51,15 @@ public class FieldLookup { return mapper; } - public Document doc() { - return doc; + public Map> fields() { + return fields; } - public void doc(Document doc) { - this.doc = doc; + /** + * Sets the post processed values. + */ + public void fields(Map> fields) { + this.fields = fields; } public void clear() { @@ -66,7 +67,7 @@ public class FieldLookup { valueLoaded = false; values.clear(); valuesLoaded = false; - doc = null; + fields = null; } public boolean isEmpty() { @@ -85,12 +86,8 @@ public class FieldLookup { } valueLoaded = true; value = null; - IndexableField field = doc.getField(mapper.names().indexName()); - if (field == null) { - return null; - } - value = mapper.value((Field) field); - return value; + List values = fields.get(mapper.names().indexName()); + return values != null ? value = values.get(0) : null; } public List getValues() { @@ -99,10 +96,6 @@ public class FieldLookup { } valuesLoaded = true; values.clear(); - IndexableField[] fields = doc.getFields(mapper.names().indexName()); - for (IndexableField field : fields) { - values.add(mapper.value((Field) field)); - } - return values; + return values = fields().get(mapper.names().indexName()); } } diff --git a/src/main/java/org/elasticsearch/search/lookup/FieldsLookup.java b/src/main/java/org/elasticsearch/search/lookup/FieldsLookup.java index 543c7e6b8dc..54f4e52fbcf 100644 --- a/src/main/java/org/elasticsearch/search/lookup/FieldsLookup.java +++ b/src/main/java/org/elasticsearch/search/lookup/FieldsLookup.java @@ -19,13 +19,14 @@ package org.elasticsearch.search.lookup; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.ElasticSearchParseException; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.lucene.document.SingleFieldVisitor; +import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; @@ -51,11 +52,12 @@ public class FieldsLookup implements Map { private final Map cachedFieldData = Maps.newHashMap(); - private final SingleFieldVisitor fieldVisitor = new SingleFieldVisitor(); + private final SingleFieldsVisitor fieldVisitor; FieldsLookup(MapperService mapperService, @Nullable String[] types) { this.mapperService = mapperService; this.types = types; + this.fieldVisitor = new SingleFieldsVisitor(null); } public void setNextReader(AtomicReaderContext context) { @@ -151,16 +153,15 @@ public class FieldsLookup implements Map { data = new FieldLookup(mapper); cachedFieldData.put(name, data); } - if (data.doc() == null) { - fieldVisitor.name(data.mapper().names().indexName()); + if (data.fields() == null) { + String fieldName = data.mapper().names().indexName(); + fieldVisitor.reset(fieldName); try { reader.document(docId, fieldVisitor); - // LUCENE 4 UPGRADE: Only one field we don't need document - data.doc(fieldVisitor.createDocument()); + fieldVisitor.postProcess(data.mapper()); + data.fields(ImmutableMap.of(name, fieldVisitor.fields().get(data.mapper().names().indexName()))); } catch (IOException e) { throw new ElasticSearchParseException("failed to load field [" + name + "]", e); - } finally { - fieldVisitor.reset(); } } return data; diff --git a/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index a1cb2d7350a..3771ed44a7a 100644 --- a/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -22,12 +22,11 @@ package org.elasticsearch.search.lookup; import com.google.common.collect.ImmutableMap; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticSearchParseException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.mapper.internal.SourceFieldVisitor; +import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor; import java.util.Collection; import java.util.List; @@ -60,13 +59,13 @@ public class SourceLookup implements Map { return source; } try { - SourceFieldVisitor sourceFieldVisitor = new SourceFieldVisitor(); + JustSourceFieldsVisitor sourceFieldVisitor = new JustSourceFieldsVisitor(); reader.document(docId, sourceFieldVisitor); - BytesRef source = sourceFieldVisitor.source(); + BytesReference source = sourceFieldVisitor.source(); if (source == null) { this.source = ImmutableMap.of(); } else { - this.source = sourceAsMap(source.bytes, source.offset, source.length); + this.source = sourceAsMap(source); } } catch (Exception e) { throw new ElasticSearchParseException("failed to parse / load source", e); diff --git a/src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java b/src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java index ef96962f0bc..8cc67cfb1f6 100644 --- a/src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java +++ b/src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java @@ -28,11 +28,14 @@ import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.util.List; + import static org.elasticsearch.client.Requests.clusterHealthRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.MatcherAssert.assertThat; @@ -278,4 +281,97 @@ public class GetActionTests extends AbstractNodesTests { assertThat((Integer) getResponse.field("int").getValue(), equalTo(42)); assertThat((String) getResponse.field("date").getValue(), equalTo("2012-11-13T15:26:14.000Z")); } + + @Test + public void testGetDocWithMultivaluedFields() throws Exception { + try { + client.admin().indices().prepareDelete("test").execute().actionGet(); + } catch (Exception e) { + // fine + } + String mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties") + .startObject("field").field("type", "string").field("store", "yes").endObject() + .endObject() + .endObject().endObject().string(); + String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type2") + .startObject("properties") + .startObject("field").field("type", "string").field("store", "yes").endObject() + .endObject() + .startObject("_source").field("enabled", false).endObject() + .endObject().endObject().string(); + client.admin().indices().prepareCreate("test") + .addMapping("type1", mapping1) + .addMapping("type2", mapping2) + .setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1)) + .execute().actionGet(); + + ClusterHealthResponse clusterHealth = client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet(); + assertThat(clusterHealth.timedOut(), equalTo(false)); + assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN)); + + GetResponse response = client.prepareGet("test", "type1", "1").execute().actionGet(); + assertThat(response.exists(), equalTo(false)); + response = client.prepareGet("test", "type2", "1").execute().actionGet(); + assertThat(response.exists(), equalTo(false)); + + client.prepareIndex("test", "type1", "1") + .setSource(jsonBuilder().startObject().field("field", "1", "2").endObject()) + .execute().actionGet(); + + client.prepareIndex("test", "type2", "1") + .setSource(jsonBuilder().startObject().field("field", "1", "2").endObject()) + .execute().actionGet(); + + response = client.prepareGet("test", "type1", "1") + .setFields("field") + .execute().actionGet(); + assertThat(response.exists(), equalTo(true)); + assertThat(response.getId(), equalTo("1")); + assertThat(response.getType(), equalTo("type1")); + assertThat(response.fields().size(), equalTo(1)); + assertThat(response.fields().get("field").values().size(), equalTo(1)); + assertThat(((List) response.fields().get("field").values().get(0)).size(), equalTo(2)); + assertThat(((List) response.fields().get("field").values().get(0)).get(0).toString(), equalTo("1")); + assertThat(((List) response.fields().get("field").values().get(0)).get(1).toString(), equalTo("2")); + + + response = client.prepareGet("test", "type2", "1") + .setFields("field") + .execute().actionGet(); + assertThat(response.exists(), equalTo(true)); + assertThat(response.getType(), equalTo("type2")); + assertThat(response.getId(), equalTo("1")); + assertThat(response.fields().size(), equalTo(1)); + assertThat(response.fields().get("field").values().size(), equalTo(1)); + assertThat(((List) response.fields().get("field").values().get(0)).size(), equalTo(2)); + assertThat(((List) response.fields().get("field").values().get(0)).get(0).toString(), equalTo("1")); + assertThat(((List) response.fields().get("field").values().get(0)).get(1).toString(), equalTo("2")); + + // Now test values being fetched from stored fields. + client.admin().indices().prepareRefresh("test").execute().actionGet(); + response = client.prepareGet("test", "type1", "1") + .setFields("field") + .execute().actionGet(); + assertThat(response.exists(), equalTo(true)); + assertThat(response.getId(), equalTo("1")); + assertThat(response.fields().size(), equalTo(1)); + assertThat(response.fields().get("field").values().size(), equalTo(1)); + assertThat(((List) response.fields().get("field").values().get(0)).size(), equalTo(2)); + assertThat(((List) response.fields().get("field").values().get(0)).get(0).toString(), equalTo("1")); + assertThat(((List) response.fields().get("field").values().get(0)).get(1).toString(), equalTo("2")); + + + response = client.prepareGet("test", "type2", "1") + .setFields("field") + .execute().actionGet(); + assertThat(response.exists(), equalTo(true)); + assertThat(response.getId(), equalTo("1")); + assertThat(response.fields().size(), equalTo(1)); + assertThat(response.fields().get("field").values().size(), equalTo(1)); + assertThat(((List) response.fields().get("field").values().get(0)).size(), equalTo(2)); + assertThat(((List) response.fields().get("field").values().get(0)).get(0).toString(), equalTo("1")); + assertThat(((List) response.fields().get("field").values().get(0)).get(1).toString(), equalTo("2")); + } + } \ No newline at end of file