honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

lucene 4: Removed the usage of Document & Field when retrieving stored fields.

This commit is contained in:
Martijn van Groningen 2012-12-06 18:18:20 +01:00
parent d947dfde2b
commit ea9a4d70cf
55 changed files with 811 additions and 964 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java
View File

@ -21,7 +21,9 @@ package org.elasticsearch.action.mlt;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
@ -33,7 +35,10 @@ import org.elasticsearch.action.support.TransportAction;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.*;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.get.GetField;
@ -47,7 +52,6 @@ import org.elasticsearch.transport.*;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import static com.google.common.collect.Sets.newHashSet;
@ -267,7 +271,7 @@ public class TransportMoreLikeThisAction extends TransportAction<MoreLikeThisReq
if (fieldMapper instanceof InternalMapper) {
return true;
}
String value = fieldMapper.valueAsString(field);
String value = fieldMapper.valueAsString(convertField(field));
if (value == null) {
return false;
}
@ -281,8 +285,20 @@ public class TransportMoreLikeThisAction extends TransportAction<MoreLikeThisReq
});
}
private Object convertField(Field field) {
if (field.stringValue() != null) {
return field.stringValue();
} else if (field.binaryValue() != null) {
return BytesRef.deepCopyOf(field.binaryValue()).bytes;
} else if (field.numericValue() != null) {
return field.numericValue();
} else {
throw new ElasticSearchIllegalStateException("Field should have either a string, numeric or binary value");
}
}
private void addMoreLikeThis(MoreLikeThisRequest request, BoolQueryBuilder boolBuilder, FieldMapper fieldMapper, Field field) {
addMoreLikeThis(request, boolBuilder, field.name(), fieldMapper.valueAsString(field));
addMoreLikeThis(request, boolBuilder, field.name(), fieldMapper.valueAsString(convertField(field)));
}
private void addMoreLikeThis(MoreLikeThisRequest request, BoolQueryBuilder boolBuilder, String fieldName, String likeText) {

63
src/main/java/org/elasticsearch/common/lucene/document/AbstractMultipleFieldsVisitor.java
View File

@ -1,63 +0,0 @@
package org.elasticsearch.common.lucene.document;
import org.apache.lucene.document.*;
import org.apache.lucene.index.FieldInfo;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
/**
*
*/
public abstract class AbstractMultipleFieldsVisitor extends BaseFieldVisitor {
protected Document doc = new Document();
@Override
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
doc.add(new StoredField(fieldInfo.name, value));
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
final FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.setStoreTermVectors(fieldInfo.hasVectors());
ft.setIndexed(fieldInfo.isIndexed());
ft.setOmitNorms(fieldInfo.omitsNorms());
ft.setIndexOptions(fieldInfo.getIndexOptions());
doc.add(new Field(fieldInfo.name, value, ft));
}
@Override
public void intField(FieldInfo fieldInfo, int value) {
doc.add(new StoredField(fieldInfo.name, value));
}
@Override
public void longField(FieldInfo fieldInfo, long value) {
doc.add(new StoredField(fieldInfo.name, value));
}
@Override
public void floatField(FieldInfo fieldInfo, float value) {
doc.add(new StoredField(fieldInfo.name, value));
}
@Override
public void doubleField(FieldInfo fieldInfo, double value) {
doc.add(new StoredField(fieldInfo.name, value));
}
@Override
public Document createDocument() {
return doc;
}
@Override
public void reset() {
if (!doc.getFields().isEmpty()) {
doc = new Document();
}
}
}

14
src/main/java/org/elasticsearch/common/lucene/document/BaseFieldVisitor.java
View File

@ -1,14 +0,0 @@
package org.elasticsearch.common.lucene.document;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.StoredFieldVisitor;
public abstract class BaseFieldVisitor extends StoredFieldVisitor {
// LUCENE 4 UPGRADE: Added for now to make everything work. Want to make use of Document as less as possible.
public abstract Document createDocument();
// LUCENE 4 UPGRADE: Added for now for compatibility with Selectors
public abstract void reset();
}

55
src/main/java/org/elasticsearch/common/lucene/document/SingleFieldVisitor.java
View File

@ -1,55 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.lucene.document;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
*
*/
public class SingleFieldVisitor extends AbstractMultipleFieldsVisitor {
private String name;
public SingleFieldVisitor() {
}
public SingleFieldVisitor(String name) {
this.name = name;
}
public void name(String name) {
this.name = name;
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (name.equals(fieldInfo.name)) {
return Status.YES;
}
return Status.NO;
}
}

63
src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
import java.util.Set;
/**
*/
public class CustomFieldsVisitor extends FieldsVisitor {
private final boolean loadAllFields;
private final boolean loadSource;
private final Set<String> fields;
public CustomFieldsVisitor(Set<String> fields, boolean loadSource) {
this.loadAllFields = false;
this.loadSource = loadSource;
this.fields = fields;
}
public CustomFieldsVisitor(boolean loadAllFields, boolean loadSource) {
this.loadAllFields = loadAllFields;
this.loadSource = loadSource;
this.fields = null;
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (loadAllFields) {
return Status.YES;
}
if (loadSource && SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return fields.contains(fieldInfo.name) ? Status.YES : Status.NO;
}
}

151
src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java Normal file
View File

@ -0,0 +1,151 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.StoredFieldVisitor;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static com.google.common.collect.Maps.newHashMap;
/**
*/
public abstract class FieldsVisitor extends StoredFieldVisitor {
protected BytesReference source;
protected Uid uid;
protected Map<String, List<Object>> fieldsValues;
public void postProcess(MapperService mapperService) {
if (fieldsValues == null || fieldsValues.isEmpty()) {
return;
}
if (uid != null) {
DocumentMapper documentMapper = mapperService.documentMapper(uid.type());
if (documentMapper != null) {
// we can derive the exact type for the mapping
postProcess(documentMapper);
return;
}
}
// can't derive exact mapping type
for (Map.Entry<String, List<Object>> entry : fieldsValues.entrySet()) {
FieldMappers fieldMappers = mapperService.indexName(entry.getKey());
if (fieldMappers == null) {
continue;
}
List<Object> fieldValues = entry.getValue();
for (int i = 0; i < fieldValues.size(); i++) {
fieldValues.set(i, fieldMappers.mapper().valueForSearch(fieldValues.get(i)));
}
}
}
public void postProcess(DocumentMapper documentMapper) {
for (Map.Entry<String, List<Object>> entry : fieldsValues.entrySet()) {
FieldMapper fieldMapper = documentMapper.mappers().indexName(entry.getKey()).mapper();
if (fieldMapper == null) {
continue;
}
List<Object> fieldValues = entry.getValue();
for (int i = 0; i < fieldValues.size(); i++) {
fieldValues.set(i, fieldMapper.valueForSearch(fieldValues.get(i)));
}
}
}
@Override
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
source = new BytesArray(value);
} else {
addValue(fieldInfo.name, new BytesArray(value));
}
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
uid = Uid.createUid(value);
} else {
addValue(fieldInfo.name, value);
}
}
@Override
public void intField(FieldInfo fieldInfo, int value) throws IOException {
addValue(fieldInfo.name, value);
}
@Override
public void longField(FieldInfo fieldInfo, long value) throws IOException {
addValue(fieldInfo.name, value);
}
@Override
public void floatField(FieldInfo fieldInfo, float value) throws IOException {
addValue(fieldInfo.name, value);
}
@Override
public void doubleField(FieldInfo fieldInfo, double value) throws IOException {
addValue(fieldInfo.name, value);
}
public BytesReference source() {
return source;
}
public Uid uid() {
return uid;
}
public Map<String, List<Object>> fields() {
return fieldsValues;
}
public void reset() {
if (fieldsValues != null) fieldsValues.clear();
source = null;
uid = null;
}
private void addValue(String name, Object value) {
if (fieldsValues == null) {
fieldsValues = newHashMap();
}
List<Object> values = fieldsValues.get(name);
if (values == null) {
values = new ArrayList<Object>(2);
fieldsValues.put(name, values);
}
values.add(value);
}
}

38
src/main/java/org/elasticsearch/index/fieldvisitor/JustSourceFieldsVisitor.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import java.io.IOException;
/**
*/
public class JustSourceFieldsVisitor extends FieldsVisitor {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return source != null ? Status.STOP : Status.NO;
}
}

38
src/main/java/org/elasticsearch/index/fieldvisitor/JustUidFieldsVisitor.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
/**
*/
public class JustUidFieldsVisitor extends FieldsVisitor {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return uid != null ? Status.STOP : Status.NO;
}
}

65
src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.index.mapper.FieldMapper;
import java.io.IOException;
import java.util.List;
/**
*/
public class SingleFieldsVisitor extends FieldsVisitor {
private String field;
public SingleFieldsVisitor(String field) {
this.field = field;
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
// TODO we can potentially skip if we processed a field, the question is if it works for multi valued fields
if (fieldInfo.name.equals(field)) {
return Status.YES;
} else {
return Status.NO;
}
}
public void reset(String field) {
this.field = field;
super.reset();
}
public void postProcess(FieldMapper mapper) {
if (fieldsValues == null) {
return;
}
List<Object> fieldValues = fieldsValues.get(mapper.names().indexName());
if (fieldValues == null) {
return;
}
for (int i = 0; i < fieldValues.size(); i++) {
fieldValues.set(i, mapper.valueForSearch(fieldValues.get(i)));
}
}
}

57
src/main/java/org/elasticsearch/index/fieldvisitor/UidAndRoutingFieldsVisitor.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
/**
*/
public class UidAndRoutingFieldsVisitor extends FieldsVisitor {
private String routing;
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
} else if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return uid != null && routing != null ? Status.STOP : Status.NO;
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) {
routing = value;
} else {
super.stringField(fieldInfo, value);
}
}
public String routing() {
return routing;
}
}

42
src/main/java/org/elasticsearch/index/fieldvisitor/UidAndSourceFieldsVisitor.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
/**
*/
public class UidAndSourceFieldsVisitor extends FieldsVisitor {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
} else if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return uid != null && source != null ? Status.STOP : Status.NO;
}
}

86
src/main/java/org/elasticsearch/index/get/ShardGetService.java
View File

@ -19,23 +19,22 @@
package org.elasticsearch.index.get;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexableField;
import com.google.common.collect.Sets;
import org.apache.lucene.index.Term;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.common.lucene.uid.UidField;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.internal.*;
import org.elasticsearch.index.mapper.selector.FieldMappersFieldVisitor;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
@ -48,6 +47,8 @@ import org.elasticsearch.search.lookup.SourceLookup;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@ -274,50 +275,23 @@ public class ShardGetService extends AbstractIndexShardComponent {
private GetResult innerGetLoadFromStoredFields(String type, String id, String[] gFields, Engine.GetResult get, DocumentMapper docMapper) {
Map<String, GetField> fields = null;
byte[] source = null;
BytesReference source = null;
UidField.DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
// LUCENE 4 UPGRADE: optimize when only a single field needs to be loaded
BaseFieldVisitor fieldVisitor = buildFieldSelectors(docMapper, gFields);
FieldsVisitor fieldVisitor = buildFieldsVisitors(gFields);
if (fieldVisitor != null) {
Document doc;
try {
docIdAndVersion.reader.reader().document(docIdAndVersion.docId, fieldVisitor);
doc = fieldVisitor.createDocument();
} catch (IOException e) {
throw new ElasticSearchException("Failed to get type [" + type + "] and id [" + id + "]", e);
}
source = extractSource(doc, docMapper);
source = fieldVisitor.source();
for (Object oField : doc.getFields()) {
Field field = (Field) oField;
String name = field.name();
Object value = null;
FieldMappers fieldMappers = docMapper.mappers().indexName(field.name());
if (fieldMappers != null) {
FieldMapper mapper = fieldMappers.mapper();
if (mapper != null) {
name = mapper.names().fullName();
value = mapper.valueForSearch(field);
}
if (fieldVisitor.fields() != null) {
fieldVisitor.postProcess(docMapper);
fields = new HashMap<String, GetField>(fieldVisitor.fields().size());
for (Map.Entry<String, List<Object>> entry : fieldVisitor.fields().entrySet()) {
fields.put(entry.getKey(), new GetField(entry.getKey(), entry.getValue()));
}
if (value == null) {
if (field.binaryValue() != null) {
value = new BytesArray(field.binaryValue());
} else {
value = field.stringValue();
}
}
if (fields == null) {
fields = newHashMapWithExpectedSize(2);
}
GetField getField = fields.get(name);
if (getField == null) {
getField = new GetField(name, new ArrayList<Object>(2));
fields.put(name, getField);
}
getField.values().add(value);
}
}
@ -368,12 +342,12 @@ public class ShardGetService extends AbstractIndexShardComponent {
}
}
return new GetResult(shardId.index().name(), type, id, get.version(), get.exists(), source == null ? null : new BytesArray(source), fields);
return new GetResult(shardId.index().name(), type, id, get.version(), get.exists(), source, fields);
}
private static BaseFieldVisitor buildFieldSelectors(DocumentMapper docMapper, String... fields) {
private static FieldsVisitor buildFieldsVisitors(String... fields) {
if (fields == null) {
return docMapper.sourceMapper().fieldSelector();
return new JustSourceFieldsVisitor();
}
// don't load anything
@ -381,28 +355,6 @@ public class ShardGetService extends AbstractIndexShardComponent {
return null;
}
FieldMappersFieldVisitor fieldVisitor = null;
for (String fieldName : fields) {
FieldMappers x = docMapper.mappers().smartName(fieldName);
if (x != null && x.mapper().stored()) {
if (fieldVisitor == null) {
fieldVisitor = new FieldMappersFieldVisitor();
}
fieldVisitor.add(x);
}
}
return fieldVisitor;
}
private static byte[] extractSource(Document doc, DocumentMapper documentMapper) {
byte[] source = null;
IndexableField sourceField = doc.getField(documentMapper.sourceMapper().names().indexName());
if (sourceField != null) {
// LUCENE 4 UPGRADE: Field instead of IndexableField?
source = documentMapper.sourceMapper().nativeValue((Field) sourceField);
doc.removeField(documentMapper.sourceMapper().names().indexName());
}
return source;
return new CustomFieldsVisitor(Sets.newHashSet(fields), false);
}
}

7
src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Filter;
@ -166,19 +165,19 @@ public interface FieldMapper<T> {
/**
* Returns the value that will be used as a result for search. Can be only of specific types... .
*/
Object valueForSearch(Field field);
Object valueForSearch(Object value);
/**
* Returns the actual value of the field.
*/
T value(Field field);
T value(Object value);
T valueFromString(String value);
/**
* Returns the actual value of the field as string.
*/
String valueAsString(Field field);
String valueAsString(Object value);
/**
* Returns the indexed value.

9
src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
View File

@ -415,8 +415,8 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
}
@Override
public Object valueForSearch(Field field) {
return valueAsString(field);
public Object valueForSearch(Object value) {
return valueAsString(value);
}
@Override
@ -424,6 +424,11 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
return value;
}
@Override
public String valueAsString(Object value) {
return String.valueOf(value);
}
@Override
public Query queryStringTermQuery(Term term) {
return null;

12
src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper.core;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.Strings;
@ -126,18 +125,17 @@ public class BinaryFieldMapper extends AbstractFieldMapper<byte[]> {
}
@Override
public Object valueForSearch(Field field) {
return value(field);
public Object valueForSearch(Object value) {
return value(value);
}
@Override
public byte[] value(Field field) {
BytesRef value = field.binaryValue();
public byte[] value(Object value) {
if (value == null) {
return null;
}
try {
return CompressorFactory.uncompressIfNeeded(new BytesArray(value)).toBytes();
return CompressorFactory.uncompressIfNeeded(new BytesArray((byte[]) value)).toBytes();
} catch (IOException e) {
throw new ElasticSearchParseException("failed to decompress source", e);
}
@ -154,7 +152,7 @@ public class BinaryFieldMapper extends AbstractFieldMapper<byte[]> {
}
@Override
public String valueAsString(Field field) {
public String valueAsString(Object value) {
return null;
}

8
src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
View File

@ -154,8 +154,8 @@ public class BooleanFieldMapper extends AbstractFieldMapper<Boolean> {
}
@Override
public Boolean value(Field field) {
return field.stringValue().charAt(0) == 'T' ? Boolean.TRUE : Boolean.FALSE;
public Boolean value(Object value) {
return valueFromString((String) value);
}
@Override
@ -164,8 +164,8 @@ public class BooleanFieldMapper extends AbstractFieldMapper<Boolean> {
}
@Override
public String valueAsString(Field field) {
return field.stringValue().charAt(0) == 'T' ? "true" : "false";
public String valueAsString(Object value) {
return ((String) value).charAt(0) == 'T' ? "true" : "false";
}
@Override

10
src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
View File

@ -32,6 +32,7 @@ import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -128,12 +129,15 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
}
@Override
public Byte value(Field field) {
BytesRef value = field.binaryValue();
public Byte value(Object value) {
if (value == null) {
return null;
}
return value.bytes[value.offset];
if (value instanceof Number) {
return ((Number) value).byteValue();
}
BytesReference bytesReference = (BytesReference) value;
return bytesReference.get(bytesReference.arrayOffset());
}
@Override

23
src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.joda.DateMathParser;
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
import org.elasticsearch.common.joda.Joda;
@ -184,12 +185,14 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
}
@Override
public Long value(Field field) {
BytesRef value = field.binaryValue();
public Long value(Object value) {
if (value == null) {
return null;
}
return Numbers.bytesToLong(value.bytes);
if (value instanceof Number) {
return ((Number) value).longValue();
}
return Numbers.bytesToLong(((BytesReference) value).array());
}
@Override
@ -199,19 +202,21 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
/**
* Dates should return as a string.
*
* @param value
*/
@Override
public Object valueForSearch(Field field) {
return valueAsString(field);
public Object valueForSearch(Object value) {
return valueAsString(value);
}
@Override
public String valueAsString(Field field) {
Long value = value(field);
if (value == null) {
public String valueAsString(Object value) {
Long val = value(value);
if (val == null) {
return null;
}
return dateTimeFormatter.printer().print(value);
return dateTimeFormatter.printer().print(val);
}
@Override

9
src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
View File

@ -32,6 +32,7 @@ import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -131,12 +132,14 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
}
@Override
public Double value(Field field) {
BytesRef value = field.binaryValue();
public Double value(Object value) {
if (value == null) {
return null;
}
return Numbers.bytesToDouble(value.bytes);
if (value instanceof Number) {
return ((Number) value).doubleValue();
}
return Numbers.bytesToDouble(((BytesReference) value).array());
}
@Override

9
src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -129,12 +130,14 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
}
@Override
public Float value(Field field) {
BytesRef value = field.binaryValue();
public Float value(Object value) {
if (value == null) {
return null;
}
return Numbers.bytesToFloat(value.bytes);
if (value instanceof Number) {
return ((Number) value).floatValue();
}
return Numbers.bytesToFloat(((BytesReference) value).array());
}
@Override

9
src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -131,12 +132,14 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
}
@Override
public Integer value(Field field) {
BytesRef value = field.binaryValue();
public Integer value(Object value) {
if (value == null) {
return null;
}
return Numbers.bytesToInt(value.bytes);
if (value instanceof Number) {
return ((Number) value).intValue();
}
return Numbers.bytesToInt(((BytesReference) value).array());
}
@Override

9
src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -131,12 +132,14 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
}
@Override
public Long value(Field field) {
BytesRef value = field.binaryValue();
public Long value(Object value) {
if (value == null) {
return null;
}
return Numbers.bytesToLong(value.bytes);
if (value instanceof Number) {
return ((Number) value).longValue();
}
return Numbers.bytesToLong(((BytesReference) value).array());
}
@Override

10
src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
View File

@ -247,15 +247,17 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
/**
* Override the default behavior (to return the string, and return the actual Number instance).
*
* @param value
*/
@Override
public Object valueForSearch(Field field) {
return value(field);
public Object valueForSearch(Object value) {
return value(value);
}
@Override
public String valueAsString(Field field) {
Number num = value(field);
public String valueAsString(Object value) {
Number num = value(value);
return num == null ? null : num.toString();
}

9
src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -131,12 +132,14 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
}
@Override
public Short value(Field field) {
BytesRef value = field.binaryValue();
public Short value(Object value) {
if (value == null) {
return null;
}
return Numbers.bytesToShort(value.bytes);
if (value instanceof Number) {
return ((Number) value).shortValue();
}
return Numbers.bytesToShort(((BytesReference) value).array());
}
@Override

9
src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
View File

@ -217,8 +217,8 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
}
@Override
public String value(Field field) {
return field.stringValue();
public String value(Object value) {
return String.valueOf(value);
}
@Override
@ -226,11 +226,6 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
return value;
}
@Override
public String valueAsString(Field field) {
return value(field);
}
@Override
public String indexedValue(String value) {
return value;

4
src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
View File

@ -176,7 +176,7 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper<String> {
}
@Override
public String value(Field field) {
public String value(Object value) {
throw new UnsupportedOperationException("GeoShape fields cannot be converted to String values");
}
@ -186,7 +186,7 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper<String> {
}
@Override
public String valueAsString(Field field) {
public String valueAsString(Object value) {
throw new UnsupportedOperationException("GeoShape fields cannot be converted to String values");
}

6
src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
View File

@ -221,7 +221,7 @@ public class AllFieldMapper extends AbstractFieldMapper<Void> implements Interna
}
@Override
public Void value(Field field) {
public Void value(Object value) {
return null;
}
@ -231,12 +231,12 @@ public class AllFieldMapper extends AbstractFieldMapper<Void> implements Interna
}
@Override
public String valueAsString(Field field) {
public String valueAsString(Object value) {
return null;
}
@Override
public Object valueForSearch(Field field) {
public Object valueForSearch(Object value) {
return null;
}

5
src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
View File

@ -132,12 +132,11 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
}
@Override
public Float value(Field field) {
BytesRef value = field.binaryValue();
public Float value(Object value) {
if (value == null) {
return null;
}
return Numbers.bytesToFloat(value.bytes);
return Numbers.bytesToFloat((byte[]) value);
}
@Override

9
src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
View File

@ -139,8 +139,8 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements Intern
}
@Override
public String value(Field field) {
return field.stringValue();
public String value(Object value) {
return String.valueOf(value);
}
@Override
@ -148,11 +148,6 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements Intern
return value;
}
@Override
public String valueAsString(Field field) {
return value(field);
}
@Override
public String indexedValue(String value) {
return value;

9
src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
View File

@ -128,8 +128,8 @@ public class IndexFieldMapper extends AbstractFieldMapper<String> implements Int
}
@Override
public String value(Field field) {
return field.stringValue();
public String value(Object value) {
return String.valueOf(value);
}
@Override
@ -137,11 +137,6 @@ public class IndexFieldMapper extends AbstractFieldMapper<String> implements Int
return value;
}
@Override
public String valueAsString(Field field) {
return value(field);
}
@Override
public String indexedValue(String value) {
return value;

13
src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
View File

@ -169,8 +169,8 @@ public class ParentFieldMapper extends AbstractFieldMapper<Uid> implements Inter
}
@Override
public Uid value(Field field) {
return Uid.createUid(field.stringValue());
public Uid value(Object value) {
return Uid.createUid(String.valueOf(value));
}
@Override
@ -179,13 +179,8 @@ public class ParentFieldMapper extends AbstractFieldMapper<Uid> implements Inter
}
@Override
public String valueAsString(Field field) {
return field.stringValue();
}
@Override
public Object valueForSearch(Field field) {
String fieldValue = field.stringValue();
public Object valueForSearch(Object value) {
String fieldValue = String.valueOf(value);
if (fieldValue == null) {
return null;
}

9
src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
View File

@ -142,8 +142,8 @@ public class RoutingFieldMapper extends AbstractFieldMapper<String> implements I
}
@Override
public String value(Field field) {
return field.stringValue();
public String value(Object value) {
return String.valueOf(value);
}
@Override
@ -151,11 +151,6 @@ public class RoutingFieldMapper extends AbstractFieldMapper<String> implements I
return value;
}
@Override
public String valueAsString(Field field) {
return value(field);
}
@Override
public String indexedValue(String value) {
return value;

18
src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
View File

@ -27,7 +27,6 @@ import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.CompressedStreamInput;
@ -36,7 +35,6 @@ import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.io.stream.CachedStreamOutput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
@ -214,10 +212,6 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
return this.enabled;
}
public BaseFieldVisitor fieldSelector() {
return new SourceFieldVisitor();
}
@Override
public void preParse(ParseContext context) throws IOException {
super.parse(context);
@ -357,13 +351,13 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
}
@Override
public byte[] value(Field field) {
byte[] value = field.binaryValue().bytes;
if (value == null) {
return value;
public byte[] value(Object value) {
BytesReference val = (BytesReference) value;
if (val == null) {
return null;
}
try {
return CompressorFactory.uncompressIfNeeded(new BytesArray(value)).toBytes();
return CompressorFactory.uncompressIfNeeded(val).toBytes();
} catch (IOException e) {
throw new ElasticSearchParseException("failed to decompress source", e);
}
@ -375,7 +369,7 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
}
@Override
public String valueAsString(Field field) {
public String valueAsString(Object value) {
throw new UnsupportedOperationException();
}

72
src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldVisitor.java
View File

@ -1,72 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.internal;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import java.io.IOException;
/**
* An optimized field selector that loads just the _source
*/
public class SourceFieldVisitor extends BaseFieldVisitor {
private BytesRef source;
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return source != null ? Status.STOP : Status.NO;
}
@Override
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
source = new BytesRef(value);
}
@Override
public Document createDocument() {
Document document = new Document();
if (source != null) {
document.add(new StoredField(SourceFieldMapper.NAME, source));
}
return document;
}
@Override
public void reset() {
source = null;
}
public BytesRef source() {
return source;
}
@Override
public String toString() {
return "source";
}
}

6
src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
View File

@ -134,7 +134,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
// Overrides valueForSearch to display live value of remaining ttl
@Override
public Object valueForSearch(Field field) {
public Object valueForSearch(Object value) {
long now;
SearchContext searchContext = SearchContext.current();
if (searchContext != null) {
@ -142,8 +142,8 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
} else {
now = System.currentTimeMillis();
}
long value = value(field);
return value - now;
long val = value(value);
return val - now;
}
// Other implementation for realtime get display

14
src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View File

@ -157,19 +157,21 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
/**
* Override the default behavior to return a timestamp
*
* @param value
*/
@Override
public Object valueForSearch(Field field) {
return value(field);
public Object valueForSearch(Object value) {
return value(value);
}
@Override
public String valueAsString(Field field) {
Long value = value(field);
if (value == null) {
public String valueAsString(Object value) {
Long val = value(value);
if (val == null) {
return null;
}
return value.toString();
return val.toString();
}
@Override

9
src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
View File

@ -112,8 +112,8 @@ public class TypeFieldMapper extends AbstractFieldMapper<String> implements Inte
}
@Override
public String value(Field field) {
return field.stringValue();
public String value(Object value) {
return String.valueOf(value);
}
@Override
@ -121,11 +121,6 @@ public class TypeFieldMapper extends AbstractFieldMapper<String> implements Inte
return value;
}
@Override
public String valueAsString(Field field) {
return value(field);
}
@Override
public String indexedValue(String value) {
return value;

9
src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
View File

@ -175,8 +175,8 @@ public class UidFieldMapper extends AbstractFieldMapper<Uid> implements Internal
}
@Override
public Uid value(Field field) {
return Uid.createUid(field.stringValue());
public Uid value(Object value) {
return Uid.createUid(String.valueOf(value));
}
@Override
@ -184,11 +184,6 @@ public class UidFieldMapper extends AbstractFieldMapper<Uid> implements Internal
return Uid.createUid(value);
}
@Override
public String valueAsString(Field field) {
return field.stringValue();
}
@Override
public String indexedValue(String value) {
return value;

19
src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
View File

@ -154,12 +154,11 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
}
@Override
public Long value(Field field) {
byte[] value = field.binaryValue().bytes;
public Long value(Object value) {
if (value == null) {
return null;
}
return Numbers.bytesToLong(value);
return Numbers.bytesToLong((byte[]) value);
}
@Override
@ -169,19 +168,21 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
/**
* IPs should return as a string.
*
* @param value
*/
@Override
public Object valueForSearch(Field field) {
return valueAsString(field);
public Object valueForSearch(Object value) {
return valueAsString(value);
}
@Override
public String valueAsString(Field field) {
Long value = value(field);
if (value == null) {
public String valueAsString(Object value) {
Long val = value(value);
if (val == null) {
return null;
}
return longToIp(value);
return longToIp(val);
}
@Override

45
src/main/java/org/elasticsearch/index/mapper/selector/AllButSourceFieldVisitor.java
View File

@ -1,45 +0,0 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.common.lucene.document.AbstractMultipleFieldsVisitor;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import java.io.IOException;
/**
* A field selector that loads all fields except the source field.
*/
public class AllButSourceFieldVisitor extends AbstractMultipleFieldsVisitor {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.NO;
}
return Status.YES;
}
@Override
public String toString() {
return "all_but_source";
}
}

57
src/main/java/org/elasticsearch/index/mapper/selector/FieldMappersFieldVisitor.java
View File

@ -1,57 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.common.lucene.document.AbstractMultipleFieldsVisitor;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMappers;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
/**
*
*/
public class FieldMappersFieldVisitor extends AbstractMultipleFieldsVisitor {
protected final Set<String> fieldsToAdd = new HashSet<String>();
public void add(String fieldName) {
fieldsToAdd.add(fieldName);
}
public void add(FieldMappers fieldMappers) {
for (FieldMapper fieldMapper : fieldMappers) {
fieldsToAdd.add(fieldMapper.names().indexName());
}
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
return fieldsToAdd.contains(fieldInfo.name) ? Status.YES : Status.NO;
}
@Override
public String toString() {
return "fields(" + fieldsToAdd + ")";
}
}

90
src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldVisitor.java
View File

@ -1,90 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
/**
* An optimized field selector that loads just the uid and the routing.
*/
public class UidAndRoutingFieldVisitor extends BaseFieldVisitor {
private String uid;
private String routing;
@Override
public Document createDocument() {
Document document = new Document();
if (uid != null) {
document.add(new StoredField(UidFieldMapper.NAME, uid));
}
if (routing != null) {
document.add(new StoredField(SourceFieldMapper.NAME, routing));
}
return document;
}
@Override
public void reset() {
uid = null;
routing = null;
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
} else if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return uid != null && routing != null ? Status.STOP : Status.NO;
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) {
routing = value;
} else if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
uid = value;
}
}
public String uid() {
return uid;
}
public String routing() {
return routing;
}
@Override
public String toString() {
return "uid_and_routing";
}
}

83
src/main/java/org/elasticsearch/index/mapper/selector/UidAndSourceFieldVisitor.java
View File

@ -1,83 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
/**
* An optimized field selector that loads just the uid and the source.
*/
public class UidAndSourceFieldVisitor extends BaseFieldVisitor {
private String uid;
private BytesRef source;
@Override
public Document createDocument() {
Document document = new Document();
if (uid != null) {
document.add(new StoredField(UidFieldMapper.NAME, uid));
}
if (source != null) {
document.add(new StoredField(SourceFieldMapper.NAME, source));
}
return document;
}
@Override
public void reset() {
source = null;
uid = null;
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
} else if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return uid != null && source != null ? Status.STOP : Status.NO;
}
@Override
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
source = new BytesRef(value);
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
uid = value;
}
@Override
public String toString() {
return "uid_and_source";
}
}

75
src/main/java/org/elasticsearch/index/mapper/selector/UidFieldVisitor.java
View File

@ -1,75 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
/**
* An optimized field selector that loads just the uid.
*/
public class UidFieldVisitor extends BaseFieldVisitor {
private String uid;
public UidFieldVisitor() {
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
uid = value;
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return uid != null ? Status.STOP : Status.NO;
}
@Override
public Document createDocument() {
Document document = new Document();
if (uid != null) {
document.add(new StoredField(UidFieldMapper.NAME, uid));
}
return document;
}
@Override
public void reset() {
uid = null;
}
public String uid() {
return uid;
}
@Override
public String toString() {
return "uid";
}
}

18
src/main/java/org/elasticsearch/index/percolator/PercolatorService.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.index.percolator;
import com.google.common.collect.Maps;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.Term;
@ -28,8 +27,6 @@ import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.TermFilter;
import org.elasticsearch.common.lucene.search.XConstantScoreQuery;
@ -37,12 +34,9 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.fieldvisitor.UidAndSourceFieldsVisitor;
import org.elasticsearch.index.indexing.IndexingOperationListener;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.mapper.selector.UidAndSourceFieldVisitor;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.IndexShardState;
@ -178,13 +172,11 @@ public class PercolatorService extends AbstractIndexComponent {
@Override
public void collect(int doc) throws IOException {
// the _source is the query
UidAndSourceFieldVisitor fieldVisitor = new UidAndSourceFieldVisitor();
reader.document(doc, fieldVisitor);
Document document = fieldVisitor.createDocument();
String id = Uid.createUid(document.get(UidFieldMapper.NAME)).id();
UidAndSourceFieldsVisitor fieldsVisitor = new UidAndSourceFieldsVisitor();
reader.document(doc, fieldsVisitor);
String id = fieldsVisitor.uid().id();
try {
BytesRef sourceVal = document.getBinaryValue(SourceFieldMapper.NAME);
queries.put(id, percolator.parseQuery(id, new BytesArray(sourceVal.bytes, sourceVal.offset, sourceVal.length)));
queries.put(id, percolator.parseQuery(id, fieldsVisitor.source()));
} catch (Exception e) {
logger.warn("failed to add query [{}]", e, id);
}

19
src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
View File

@ -41,12 +41,13 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.fieldvisitor.UidAndRoutingFieldsVisitor;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMappers;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.mapper.selector.UidAndRoutingFieldVisitor;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.shard.IndexShardState;
import org.elasticsearch.index.shard.service.IndexShard;
@ -188,7 +189,7 @@ public class IndicesTTLService extends AbstractLifecycleComponent<IndicesTTLServ
Engine.Searcher searcher = shardToPurge.searcher();
try {
logger.debug("[{}][{}] purging shard", shardToPurge.routingEntry().index(), shardToPurge.routingEntry().id());
ExpiredDocsCollector expiredDocsCollector = new ExpiredDocsCollector();
ExpiredDocsCollector expiredDocsCollector = new ExpiredDocsCollector(shardToPurge.routingEntry().index());
searcher.searcher().search(query, expiredDocsCollector);
List<DocToPurge> docsToPurge = expiredDocsCollector.getDocsToPurge();
BulkRequestBuilder bulkRequest = client.prepareBulk();
@ -220,10 +221,12 @@ public class IndicesTTLService extends AbstractLifecycleComponent<IndicesTTLServ
}
private class ExpiredDocsCollector extends Collector {
private final MapperService mapperService;
private AtomicReaderContext context;
private List<DocToPurge> docsToPurge = new ArrayList<DocToPurge>();
public ExpiredDocsCollector() {
public ExpiredDocsCollector(String index) {
mapperService = indicesService.indexService(index).mapperService();
}
public void setScorer(Scorer scorer) {
@ -235,11 +238,11 @@ public class IndicesTTLService extends AbstractLifecycleComponent<IndicesTTLServ
public void collect(int doc) {
try {
UidAndRoutingFieldVisitor fieldVisitor = new UidAndRoutingFieldVisitor();
context.reader().document(doc, fieldVisitor);
String uid = fieldVisitor.uid();
long version = UidField.loadVersion(context, new Term(UidFieldMapper.NAME, uid));
docsToPurge.add(new DocToPurge(Uid.typeFromUid(uid), Uid.idFromUid(uid), version, fieldVisitor.routing()));
UidAndRoutingFieldsVisitor fieldsVisitor = new UidAndRoutingFieldsVisitor();
context.reader().document(doc, fieldsVisitor);
Uid uid = fieldsVisitor.uid();
long version = UidField.loadVersion(context, new Term(UidFieldMapper.NAME, uid.toString()));
docsToPurge.add(new DocToPurge(uid.type(), uid.id(), version, fieldsVisitor.routing()));
} catch (Exception e) {
logger.trace("failed to collect doc", e);
}

166
src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
View File

@ -20,27 +20,16 @@
package org.elasticsearch.search.fetch;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.ReaderUtil;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
import org.elasticsearch.index.fieldvisitor.JustUidFieldsVisitor;
import org.elasticsearch.index.fieldvisitor.UidAndSourceFieldsVisitor;
import org.elasticsearch.index.mapper.FieldMappers;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.mapper.selector.*;
import org.elasticsearch.index.mapper.selector.AllButSourceFieldVisitor;
import org.elasticsearch.index.mapper.selector.FieldMappersFieldVisitor;
import org.elasticsearch.indices.TypeMissingException;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.SearchPhase;
@ -56,10 +45,9 @@ import org.elasticsearch.search.internal.InternalSearchHits;
import org.elasticsearch.search.internal.SearchContext;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import static com.google.common.collect.Lists.newArrayList;
/**
*
@ -89,28 +77,25 @@ public class FetchPhase implements SearchPhase {
}
public void execute(SearchContext context) {
BaseFieldVisitor fieldVisitor;
FieldsVisitor fieldsVisitor;
List<String> extractFieldNames = null;
boolean sourceRequested = false;
if (!context.hasFieldNames()) {
if (context.hasPartialFields()) {
// partial fields need the source, so fetch it, but don't return it
fieldVisitor = new UidAndSourceFieldVisitor();
sourceRequested = false;
fieldsVisitor = new UidAndSourceFieldsVisitor();
} else if (context.hasScriptFields()) {
// we ask for script fields, and no field names, don't load the source
fieldVisitor = new UidFieldVisitor();
sourceRequested = false;
fieldsVisitor = new JustUidFieldsVisitor();
} else {
fieldVisitor = new UidAndSourceFieldVisitor();
sourceRequested = true;
fieldsVisitor = new UidAndSourceFieldsVisitor();
}
} else if (context.fieldNames().isEmpty()) {
fieldVisitor = new UidFieldVisitor();
sourceRequested = false;
fieldsVisitor = new JustUidFieldsVisitor();
} else {
boolean loadAllStored = false;
FieldMappersFieldVisitor fieldVisitorMapper = null;
Set<String> fieldNames = null;
for (String fieldName : context.fieldNames()) {
if (fieldName.equals("*")) {
loadAllStored = true;
@ -122,101 +107,52 @@ public class FetchPhase implements SearchPhase {
}
FieldMappers x = context.smartNameFieldMappers(fieldName);
if (x != null && x.mapper().stored()) {
if (fieldVisitorMapper == null) {
fieldVisitorMapper = new FieldMappersFieldVisitor();
if (fieldNames == null) {
fieldNames = new HashSet<String>();
}
fieldVisitorMapper.add(x);
fieldNames.add(x.mapper().names().indexName());
} else {
if (extractFieldNames == null) {
extractFieldNames = Lists.newArrayList();
extractFieldNames = newArrayList();
}
extractFieldNames.add(fieldName);
}
}
if (loadAllStored) {
if (sourceRequested || extractFieldNames != null) {
fieldVisitor = null; // load everything, including _source
fieldsVisitor = new CustomFieldsVisitor(true, true); // load everything, including _source
} else {
fieldVisitor = new AllButSourceFieldVisitor();
fieldsVisitor = new CustomFieldsVisitor(true, false);
}
} else if (fieldVisitorMapper != null) {
// we are asking specific stored fields, just add the UID and be done
fieldVisitorMapper.add(UidFieldMapper.NAME);
if (extractFieldNames != null || sourceRequested) {
fieldVisitorMapper.add(SourceFieldMapper.NAME);
}
fieldVisitor = fieldVisitorMapper;
} else if (fieldNames != null) {
boolean loadSource = extractFieldNames != null || sourceRequested;
fieldsVisitor = new CustomFieldsVisitor(fieldNames, loadSource);
} else if (extractFieldNames != null || sourceRequested) {
fieldVisitor = new UidAndSourceFieldVisitor();
fieldsVisitor = new UidAndSourceFieldsVisitor();
} else {
fieldVisitor = new UidFieldVisitor();
fieldsVisitor = new JustUidFieldsVisitor();
}
}
InternalSearchHit[] hits = new InternalSearchHit[context.docIdsToLoadSize()];
for (int index = 0; index < context.docIdsToLoadSize(); index++) {
int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index];
Document doc = loadDocument(context, fieldVisitor, docId);
Uid uid = extractUid(context, doc, fieldVisitor);
DocumentMapper documentMapper = context.mapperService().documentMapper(uid.type());
loadStoredFields(context, fieldsVisitor, docId);
fieldsVisitor.postProcess(context.mapperService());
if (documentMapper == null) {
throw new TypeMissingException(new Index(context.shardTarget().index()), uid.type(), "failed to find type loaded for doc [" + uid.id() + "]");
Map<String, SearchHitField> searchFields = null;
if (fieldsVisitor.fields() != null) {
searchFields = new HashMap<String, SearchHitField>(fieldsVisitor.fields().size());
for (Map.Entry<String, List<Object>> entry : fieldsVisitor.fields().entrySet()) {
searchFields.put(entry.getKey(), new InternalSearchHitField(entry.getKey(), entry.getValue()));
}
}
byte[] source = extractSource(doc, documentMapper);
InternalSearchHit searchHit = new InternalSearchHit(docId, fieldsVisitor.uid().id(), fieldsVisitor.uid().type(), sourceRequested ? fieldsVisitor.source() : null, searchFields);
// get the version
InternalSearchHit searchHit = new InternalSearchHit(docId, uid.id(), uid.type(), sourceRequested ? source : null, null);
hits[index] = searchHit;
for (Object oField : doc.getFields()) {
IndexableField field = (IndexableField) oField;
String name = field.name();
// ignore UID, we handled it above
if (name.equals(UidFieldMapper.NAME)) {
continue;
}
// ignore source, we handled it above
if (name.equals(SourceFieldMapper.NAME)) {
continue;
}
Object value = null;
FieldMappers fieldMappers = documentMapper.mappers().indexName(field.name());
if (fieldMappers != null) {
FieldMapper mapper = fieldMappers.mapper();
if (mapper != null) {
name = mapper.names().fullName();
// LUCENE 4 UPGRADE: do we really need to use Field instead of IndexableField?
value = mapper.valueForSearch((Field) field);
}
}
if (value == null) {
if (field.binaryValue() != null) {
value = new BytesArray(field.binaryValue());
} else {
value = field.stringValue();
}
}
if (searchHit.fieldsOrNull() == null) {
searchHit.fields(new HashMap<String, SearchHitField>(2));
}
SearchHitField hitField = searchHit.fields().get(name);
if (hitField == null) {
hitField = new InternalSearchHitField(name, new ArrayList<Object>(2));
searchHit.fields().put(name, hitField);
}
hitField.values().add(value);
}
int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves());
AtomicReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex);
int subDoc = docId - subReaderContext.docBase;
@ -224,8 +160,8 @@ public class FetchPhase implements SearchPhase {
// go over and extract fields that are not mapped / stored
context.lookup().setNextReader(subReaderContext);
context.lookup().setNextDocId(subDoc);
if (source != null) {
context.lookup().source().setNextSource(new BytesArray(source));
if (searchHit.source() != null) {
context.lookup().source().setNextSource(new BytesArray(searchHit.source()));
}
if (extractFieldNames != null) {
for (String extractFieldName : extractFieldNames) {
@ -248,7 +184,7 @@ public class FetchPhase implements SearchPhase {
for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
if (fetchSubPhase.hitExecutionNeeded(context)) {
hitContext.reset(searchHit, subReaderContext, subDoc, context.searcher().getIndexReader(), docId, doc);
hitContext.reset(searchHit, subReaderContext, subDoc, context.searcher().getIndexReader(), docId, fieldsVisitor);
fetchSubPhase.hitExecute(context, hitContext);
}
}
@ -263,36 +199,10 @@ public class FetchPhase implements SearchPhase {
context.fetchResult().hits(new InternalSearchHits(hits, context.queryResult().topDocs().totalHits, context.queryResult().topDocs().getMaxScore()));
}
private byte[] extractSource(Document doc, DocumentMapper documentMapper) {
IndexableField sourceField = doc.getField(SourceFieldMapper.NAME);
if (sourceField != null) {
//LUCENE 4 UPGRADE: I think all sourceFields are of type Field
return documentMapper.sourceMapper().nativeValue((Field) sourceField);
}
return null;
}
private Uid extractUid(SearchContext context, Document doc, @Nullable BaseFieldVisitor fieldVisitor) {
String sUid = doc.get(UidFieldMapper.NAME);
if (sUid != null) {
return Uid.createUid(sUid);
}
// no type, nothing to do (should not really happen)
List<String> fieldNames = new ArrayList<String>();
for (IndexableField field : doc.getFields()) {
fieldNames.add(field.name());
}
throw new FetchPhaseExecutionException(context, "Failed to load uid from the index, missing internal _uid field, current fields in the doc [" + fieldNames + "], selector [" + fieldVisitor + "]");
}
private Document loadDocument(SearchContext context, @Nullable BaseFieldVisitor fieldVisitor, int docId) {
private void loadStoredFields(SearchContext context, FieldsVisitor fieldVisitor, int docId) {
fieldVisitor.reset();
try {
if (fieldVisitor == null) {
return context.searcher().doc(docId);
}
fieldVisitor.reset();
context.searcher().doc(docId, fieldVisitor);
return fieldVisitor.createDocument();
} catch (IOException e) {
throw new FetchPhaseExecutionException(context, "Failed to fetch doc id [" + docId + "]", e);
}

12
src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
View File

@ -20,11 +20,11 @@
package org.elasticsearch.search.fetch;
import com.google.common.collect.Maps;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.internal.InternalSearchHit;
import org.elasticsearch.search.internal.SearchContext;
@ -42,16 +42,16 @@ public interface FetchSubPhase {
private int topLevelDocId;
private AtomicReaderContext readerContext;
private int docId;
private Document doc;
private FieldsVisitor fieldVisitor;
private Map<String, Object> cache;
public void reset(InternalSearchHit hit, AtomicReaderContext context, int docId, IndexReader topLevelReader, int topLevelDocId, Document doc) {
public void reset(InternalSearchHit hit, AtomicReaderContext context, int docId, IndexReader topLevelReader, int topLevelDocId, FieldsVisitor fieldVisitor) {
this.hit = hit;
this.readerContext = context;
this.docId = docId;
this.topLevelReader = topLevelReader;
this.topLevelDocId = topLevelDocId;
this.doc = doc;
this.fieldVisitor = fieldVisitor;
}
public InternalSearchHit hit() {
@ -78,8 +78,8 @@ public interface FetchSubPhase {
return topLevelDocId;
}
public Document doc() {
return doc;
public FieldsVisitor fieldVisitor() {
return fieldVisitor;
}
public Map<String, Object> cache() {

5
src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java
View File

@ -60,7 +60,10 @@ public class VersionFetchSubPhase implements FetchSubPhase {
// it might make sense to cache the TermDocs on a shared fetch context and just skip here)
// it is going to mean we work on the high level multi reader and not the lower level reader as is
// the case below...
long version = UidField.loadVersion(hitContext.readerContext(), new Term(UidFieldMapper.NAME, hitContext.doc().get(UidFieldMapper.NAME)));
long version = UidField.loadVersion(
hitContext.readerContext(),
new Term(UidFieldMapper.NAME, hitContext.fieldVisitor().uid().toString())
);
if (version < 0) {
version = -1;
}

15
src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
View File

@ -20,11 +20,10 @@
package org.elasticsearch.search.highlight;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.highlight.*;
import org.apache.lucene.search.highlight.Formatter;
@ -34,10 +33,10 @@ import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FastStringReader;
import org.elasticsearch.common.lucene.document.SingleFieldVisitor;
import org.elasticsearch.common.lucene.search.vectorhighlight.SimpleBoundaryScanner2;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.StringText;
import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
@ -176,15 +175,9 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
List<Object> textsToHighlight;
if (mapper.stored()) {
try {
SingleFieldVisitor fieldVisitor = new SingleFieldVisitor(mapper.names().indexName());
CustomFieldsVisitor fieldVisitor = new CustomFieldsVisitor(ImmutableSet.of(mapper.names().indexName()), false);
hitContext.reader().document(hitContext.docId(), fieldVisitor);
Document doc = fieldVisitor.createDocument();
textsToHighlight = new ArrayList<Object>(doc.getFields().size());
for (IndexableField docField : doc.getFields()) {
if (docField.stringValue() != null) {
textsToHighlight.add(docField.stringValue());
}
}
textsToHighlight = fieldVisitor.fields().get(mapper.names().indexName());
} catch (Exception e) {
throw new FetchPhaseExecutionException(context, "Failed to highlight field [" + field.field() + "]", e);
}

4
src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
View File

@ -91,11 +91,11 @@ public class InternalSearchHit implements SearchHit {
}
public InternalSearchHit(int docId, String id, String type, byte[] source, Map<String, SearchHitField> fields) {
public InternalSearchHit(int docId, String id, String type, BytesReference source, Map<String, SearchHitField> fields) {
this.docId = docId;
this.id = id;
this.type = type;
this.source = source == null ? null : new BytesArray(source);
this.source = source;
this.fields = fields;
}

33
src/main/java/org/elasticsearch/search/lookup/FieldLookup.java
View File

@ -19,13 +19,11 @@
package org.elasticsearch.search.lookup;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.index.mapper.FieldMapper;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
*
@ -35,7 +33,7 @@ public class FieldLookup {
// we can cached mapper completely per name, since its on an index/shard level (the lookup, and it does not change within the scope of a search request)
private final FieldMapper mapper;
private Document doc;
private Map<String, List<Object>> fields;
private Object value;
@ -53,12 +51,15 @@ public class FieldLookup {
return mapper;
}
public Document doc() {
return doc;
public Map<String, List<Object>> fields() {
return fields;
}
public void doc(Document doc) {
this.doc = doc;
/**
* Sets the post processed values.
*/
public void fields(Map<String, List<Object>> fields) {
this.fields = fields;
}
public void clear() {
@ -66,7 +67,7 @@ public class FieldLookup {
valueLoaded = false;
values.clear();
valuesLoaded = false;
doc = null;
fields = null;
}
public boolean isEmpty() {
@ -85,12 +86,8 @@ public class FieldLookup {
}
valueLoaded = true;
value = null;
IndexableField field = doc.getField(mapper.names().indexName());
if (field == null) {
return null;
}
value = mapper.value((Field) field);
return value;
List<Object> values = fields.get(mapper.names().indexName());
return values != null ? value = values.get(0) : null;
}
public List<Object> getValues() {
@ -99,10 +96,6 @@ public class FieldLookup {
}
valuesLoaded = true;
values.clear();
IndexableField[] fields = doc.getFields(mapper.names().indexName());
for (IndexableField field : fields) {
values.add(mapper.value((Field) field));
}
return values;
return values = fields().get(mapper.names().indexName());
}
}

17
src/main/java/org/elasticsearch/search/lookup/FieldsLookup.java
View File

@ -19,13 +19,14 @@
package org.elasticsearch.search.lookup;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.lucene.document.SingleFieldVisitor;
import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
@ -51,11 +52,12 @@ public class FieldsLookup implements Map {
private final Map<String, FieldLookup> cachedFieldData = Maps.newHashMap();
private final SingleFieldVisitor fieldVisitor = new SingleFieldVisitor();
private final SingleFieldsVisitor fieldVisitor;
FieldsLookup(MapperService mapperService, @Nullable String[] types) {
this.mapperService = mapperService;
this.types = types;
this.fieldVisitor = new SingleFieldsVisitor(null);
}
public void setNextReader(AtomicReaderContext context) {
@ -151,16 +153,15 @@ public class FieldsLookup implements Map {
data = new FieldLookup(mapper);
cachedFieldData.put(name, data);
}
if (data.doc() == null) {
fieldVisitor.name(data.mapper().names().indexName());
if (data.fields() == null) {
String fieldName = data.mapper().names().indexName();
fieldVisitor.reset(fieldName);
try {
reader.document(docId, fieldVisitor);
// LUCENE 4 UPGRADE: Only one field we don't need document
data.doc(fieldVisitor.createDocument());
fieldVisitor.postProcess(data.mapper());
data.fields(ImmutableMap.of(name, fieldVisitor.fields().get(data.mapper().names().indexName())));
} catch (IOException e) {
throw new ElasticSearchParseException("failed to load field [" + name + "]", e);
} finally {
fieldVisitor.reset();
}
}
return data;

9
src/main/java/org/elasticsearch/search/lookup/SourceLookup.java
View File

@ -22,12 +22,11 @@ package org.elasticsearch.search.lookup;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.mapper.internal.SourceFieldVisitor;
import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor;
import java.util.Collection;
import java.util.List;
@ -60,13 +59,13 @@ public class SourceLookup implements Map {
return source;
}
try {
SourceFieldVisitor sourceFieldVisitor = new SourceFieldVisitor();
JustSourceFieldsVisitor sourceFieldVisitor = new JustSourceFieldsVisitor();
reader.document(docId, sourceFieldVisitor);
BytesRef source = sourceFieldVisitor.source();
BytesReference source = sourceFieldVisitor.source();
if (source == null) {
this.source = ImmutableMap.of();
} else {
this.source = sourceAsMap(source.bytes, source.offset, source.length);
this.source = sourceAsMap(source);
}
} catch (Exception e) {
throw new ElasticSearchParseException("failed to parse / load source", e);

96
src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java
View File

@ -28,11 +28,14 @@ import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.test.integration.AbstractNodesTests;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.List;
import static org.elasticsearch.client.Requests.clusterHealthRequest;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
@ -278,4 +281,97 @@ public class GetActionTests extends AbstractNodesTests {
assertThat((Integer) getResponse.field("int").getValue(), equalTo(42));
assertThat((String) getResponse.field("date").getValue(), equalTo("2012-11-13T15:26:14.000Z"));
}
@Test
public void testGetDocWithMultivaluedFields() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// fine
}
String mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties")
.startObject("field").field("type", "string").field("store", "yes").endObject()
.endObject()
.endObject().endObject().string();
String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type2")
.startObject("properties")
.startObject("field").field("type", "string").field("store", "yes").endObject()
.endObject()
.startObject("_source").field("enabled", false).endObject()
.endObject().endObject().string();
client.admin().indices().prepareCreate("test")
.addMapping("type1", mapping1)
.addMapping("type2", mapping2)
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
.execute().actionGet();
ClusterHealthResponse clusterHealth = client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
assertThat(clusterHealth.timedOut(), equalTo(false));
assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN));
GetResponse response = client.prepareGet("test", "type1", "1").execute().actionGet();
assertThat(response.exists(), equalTo(false));
response = client.prepareGet("test", "type2", "1").execute().actionGet();
assertThat(response.exists(), equalTo(false));
client.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject().field("field", "1", "2").endObject())
.execute().actionGet();
client.prepareIndex("test", "type2", "1")
.setSource(jsonBuilder().startObject().field("field", "1", "2").endObject())
.execute().actionGet();
response = client.prepareGet("test", "type1", "1")
.setFields("field")
.execute().actionGet();
assertThat(response.exists(), equalTo(true));
assertThat(response.getId(), equalTo("1"));
assertThat(response.getType(), equalTo("type1"));
assertThat(response.fields().size(), equalTo(1));
assertThat(response.fields().get("field").values().size(), equalTo(1));
assertThat(((List) response.fields().get("field").values().get(0)).size(), equalTo(2));
assertThat(((List) response.fields().get("field").values().get(0)).get(0).toString(), equalTo("1"));
assertThat(((List) response.fields().get("field").values().get(0)).get(1).toString(), equalTo("2"));
response = client.prepareGet("test", "type2", "1")
.setFields("field")
.execute().actionGet();
assertThat(response.exists(), equalTo(true));
assertThat(response.getType(), equalTo("type2"));
assertThat(response.getId(), equalTo("1"));
assertThat(response.fields().size(), equalTo(1));
assertThat(response.fields().get("field").values().size(), equalTo(1));
assertThat(((List) response.fields().get("field").values().get(0)).size(), equalTo(2));
assertThat(((List) response.fields().get("field").values().get(0)).get(0).toString(), equalTo("1"));
assertThat(((List) response.fields().get("field").values().get(0)).get(1).toString(), equalTo("2"));
// Now test values being fetched from stored fields.
client.admin().indices().prepareRefresh("test").execute().actionGet();
response = client.prepareGet("test", "type1", "1")
.setFields("field")
.execute().actionGet();
assertThat(response.exists(), equalTo(true));
assertThat(response.getId(), equalTo("1"));
assertThat(response.fields().size(), equalTo(1));
assertThat(response.fields().get("field").values().size(), equalTo(1));
assertThat(((List) response.fields().get("field").values().get(0)).size(), equalTo(2));
assertThat(((List) response.fields().get("field").values().get(0)).get(0).toString(), equalTo("1"));
assertThat(((List) response.fields().get("field").values().get(0)).get(1).toString(), equalTo("2"));
response = client.prepareGet("test", "type2", "1")
.setFields("field")
.execute().actionGet();
assertThat(response.exists(), equalTo(true));
assertThat(response.getId(), equalTo("1"));
assertThat(response.fields().size(), equalTo(1));
assertThat(response.fields().get("field").values().size(), equalTo(1));
assertThat(((List) response.fields().get("field").values().get(0)).size(), equalTo(2));
assertThat(((List) response.fields().get("field").values().get(0)).get(0).toString(), equalTo("1"));
assertThat(((List) response.fields().get("field").values().get(0)).get(1).toString(), equalTo("2"));
}
}