honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

lucene 4: Moved from FieldSelectors to FieldVisitors.

This commit is contained in:
Martijn van Groningen 2012-10-29 14:11:58 +01:00 committed by Shay Banon
parent 77cbe0a26b
commit d8d7498292
23 changed files with 614 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/org/elasticsearch/common/lucene/document/BaseFieldVisitor.java Normal file
View File

@ -0,0 +1,14 @@
package org.elasticsearch.common.lucene.document;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.StoredFieldVisitor;
public abstract class BaseFieldVisitor extends StoredFieldVisitor {
// LUCENE 4 UPGRADE: Some field visitors need to be cleared before they can be reused. Maybe a better way.
public abstract void reset();
// LUCENE 4 UPGRADE: Added for now to make everything work. Want to make use of Document as less as possible.
public abstract Document createDocument();
}

85
src/main/java/org/elasticsearch/common/lucene/document/MultipleFieldsVisitor.java Normal file
View File

@ -0,0 +1,85 @@
package org.elasticsearch.common.lucene.document;
import org.apache.lucene.document.*;
import org.apache.lucene.index.FieldInfo;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
/**
*
*/
public class MultipleFieldsVisitor extends BaseFieldVisitor {
protected Document doc = new Document();
protected final Set<String> fieldsToAdd;
/** Load only fields named in the provided <code>Set&lt;String&gt;</code>. */
public MultipleFieldsVisitor(Set<String> fieldsToAdd) {
this.fieldsToAdd = fieldsToAdd;
}
/** Load only fields named in the provided <code>Set&lt;String&gt;</code>. */
public MultipleFieldsVisitor(String... fields) {
fieldsToAdd = new HashSet<String>(fields.length);
for(String field : fields) {
fieldsToAdd.add(field);
}
}
/** Load all stored fields. */
public MultipleFieldsVisitor() {
this.fieldsToAdd = null;
}
@Override
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
doc.add(new StoredField(fieldInfo.name, value));
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
final FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.setStoreTermVectors(fieldInfo.hasVectors());
ft.setIndexed(fieldInfo.isIndexed());
ft.setOmitNorms(fieldInfo.omitsNorms());
ft.setIndexOptions(fieldInfo.getIndexOptions());
doc.add(new Field(fieldInfo.name, value, ft));
}
@Override
public void intField(FieldInfo fieldInfo, int value) {
doc.add(new StoredField(fieldInfo.name, value));
}
@Override
public void longField(FieldInfo fieldInfo, long value) {
doc.add(new StoredField(fieldInfo.name, value));
}
@Override
public void floatField(FieldInfo fieldInfo, float value) {
doc.add(new StoredField(fieldInfo.name, value));
}
@Override
public void doubleField(FieldInfo fieldInfo, double value) {
doc.add(new StoredField(fieldInfo.name, value));
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
return fieldsToAdd == null || fieldsToAdd.contains(fieldInfo.name) ? Status.YES : Status.NO;
}
@Override
public void reset() {
doc = null;
}
@Override
public Document createDocument() {
return doc;
}
}

29
src/main/java/org/elasticsearch/common/lucene/document/ResetFieldSelector.java
View File

@ -1,29 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.lucene.document;
import org.apache.lucene.document.FieldSelector;
/**
*/
public interface ResetFieldSelector extends FieldSelector {
void reset();
}

53
src/main/java/org/elasticsearch/common/lucene/document/SingleFieldSelector.java
View File

@ -1,53 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.lucene.document;
import org.apache.lucene.document.FieldSelectorResult;
/**
*
*/
public class SingleFieldSelector implements ResetFieldSelector {
private String name;
public SingleFieldSelector() {
}
public SingleFieldSelector(String name) {
this.name = name;
}
public void name(String name) {
this.name = name;
}
@Override
public FieldSelectorResult accept(String fieldName) {
if (name.equals(fieldName)) {
return FieldSelectorResult.LOAD;
}
return FieldSelectorResult.NO_LOAD;
}
@Override
public void reset() {
}
}

89
src/main/java/org/elasticsearch/common/lucene/document/SingleFieldVisitor.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.lucene.document;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
*
*/
public class SingleFieldVisitor extends BaseFieldVisitor {
private String name;
private List<String> values;
public SingleFieldVisitor() {
}
public SingleFieldVisitor(String name) {
this.name = name;
}
public void name(String name) {
this.name = name;
}
@Override
public Document createDocument() {
Document document = new Document();
for (String value : values) {
document.add(new StoredField(name, value));
}
return document;
}
public String value() {
return values.get(0);
}
public List<String> values() {
return values;
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (name.equals(fieldInfo.name)) {
return Status.YES;
}
return values != null ? Status.STOP : Status.NO;
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
if (fieldInfo.name.equals(name)) {
if (values == null) {
values = new ArrayList<String>();
}
values.add(value);
}
}
@Override
public void reset() {
values = null;
}
}

45
src/main/java/org/elasticsearch/index/get/ShardGetService.java
View File

@ -20,11 +20,13 @@
package org.elasticsearch.index.get;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.document.ResetFieldSelector;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.common.lucene.uid.UidField;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.metrics.MeanMetric;
@ -33,7 +35,7 @@ import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.internal.*;
import org.elasticsearch.index.mapper.selector.FieldMappersFieldSelector;
import org.elasticsearch.index.mapper.selector.FieldMappersFieldVisitor;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
@ -140,7 +142,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
Engine.GetResult get = null;
if (type == null || type.equals("_all")) {
for (String typeX : mapperService.types()) {
get = indexShard.get(new Engine.Get(realtime, UidFieldMapper.TERM_FACTORY.createTerm(Uid.createUid(typeX, id))).loadSource(loadSource));
get = indexShard.get(new Engine.Get(realtime, new Term(UidFieldMapper.NAME, Uid.createUid(typeX, id))).loadSource(loadSource));
if (get.exists()) {
type = typeX;
break;
@ -156,7 +158,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
return new GetResult(shardId.index().name(), type, id, -1, false, null, null);
}
} else {
get = indexShard.get(new Engine.Get(realtime, UidFieldMapper.TERM_FACTORY.createTerm(Uid.createUid(type, id))).loadSource(loadSource));
get = indexShard.get(new Engine.Get(realtime, new Term(UidFieldMapper.NAME, Uid.createUid(type, id))).loadSource(loadSource));
if (!get.exists()) {
get.release();
return new GetResult(shardId.index().name(), type, id, -1, false, null, null);
@ -277,19 +279,21 @@ public class ShardGetService extends AbstractIndexShardComponent {
Map<String, GetField> fields = null;
byte[] source = null;
UidField.DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
ResetFieldSelector fieldSelector = buildFieldSelectors(docMapper, gFields);
if (fieldSelector != null) {
fieldSelector.reset();
// LUCENE 4 UPGRADE: optimize when only a single field needs to be loaded
BaseFieldVisitor fieldVisitor = buildFieldSelectors(docMapper, gFields);
if (fieldVisitor != null) {
fieldVisitor.reset();
Document doc;
try {
doc = docIdAndVersion.reader.document(docIdAndVersion.docId, fieldSelector);
docIdAndVersion.reader.reader().document(docIdAndVersion.docId, fieldVisitor);
doc = fieldVisitor.createDocument();
} catch (IOException e) {
throw new ElasticSearchException("Failed to get type [" + type + "] and id [" + id + "]", e);
}
source = extractSource(doc, docMapper);
for (Object oField : doc.getFields()) {
Fieldable field = (Fieldable) oField;
Field field = (Field) oField;
String name = field.name();
Object value = null;
FieldMappers fieldMappers = docMapper.mappers().indexName(field.name());
@ -301,8 +305,8 @@ public class ShardGetService extends AbstractIndexShardComponent {
}
}
if (value == null) {
if (field.isBinary()) {
value = new BytesArray(field.getBinaryValue(), field.getBinaryOffset(), field.getBinaryLength());
if (field.binaryValue() != null) {
value = new BytesArray(field.binaryValue());
} else {
value = field.stringValue();
}
@ -371,7 +375,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
return new GetResult(shardId.index().name(), type, id, get.version(), get.exists(), source == null ? null : new BytesArray(source), fields);
}
private static ResetFieldSelector buildFieldSelectors(DocumentMapper docMapper, String... fields) {
private static BaseFieldVisitor buildFieldSelectors(DocumentMapper docMapper, String... fields) {
if (fields == null) {
return docMapper.sourceMapper().fieldSelector();
}
@ -381,25 +385,26 @@ public class ShardGetService extends AbstractIndexShardComponent {
return null;
}
FieldMappersFieldSelector fieldSelector = null;
FieldMappersFieldVisitor fieldVisitor = null;
for (String fieldName : fields) {
FieldMappers x = docMapper.mappers().smartName(fieldName);
if (x != null && x.mapper().stored()) {
if (fieldSelector == null) {
fieldSelector = new FieldMappersFieldSelector();
if (fieldVisitor == null) {
fieldVisitor = new FieldMappersFieldVisitor();
}
fieldSelector.add(x);
fieldVisitor.add(x);
}
}
return fieldSelector;
return fieldVisitor;
}
private static byte[] extractSource(Document doc, DocumentMapper documentMapper) {
byte[] source = null;
Fieldable sourceField = doc.getFieldable(documentMapper.sourceMapper().names().indexName());
IndexableField sourceField = doc.getField(documentMapper.sourceMapper().names().indexName());
if (sourceField != null) {
source = documentMapper.sourceMapper().nativeValue(sourceField);
// LUCENE 4 UPGRADE: Field instead of IndexableField?
source = documentMapper.sourceMapper().nativeValue((Field) sourceField);
doc.removeField(documentMapper.sourceMapper().names().indexName());
}
return source;

6
src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
View File

@ -36,7 +36,7 @@ import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.io.stream.CachedStreamOutput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.document.ResetFieldSelector;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
@ -212,8 +212,8 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
return this.enabled;
}
public ResetFieldSelector fieldSelector() {
return SourceFieldSelector.INSTANCE;
public BaseFieldVisitor fieldSelector() {
return SourceFieldVisitor.INSTANCE;
}
@Override

52
src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldSelector.java
View File

@ -1,52 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.internal;
import org.apache.lucene.document.FieldSelectorResult;
import org.elasticsearch.common.lucene.document.ResetFieldSelector;
/**
* An optimized field selector that loads just the uid.
*/
public class SourceFieldSelector implements ResetFieldSelector {
public static final SourceFieldSelector INSTANCE = new SourceFieldSelector();
private SourceFieldSelector() {
}
@Override
public FieldSelectorResult accept(String fieldName) {
if (SourceFieldMapper.NAME.equals(fieldName)) {
return FieldSelectorResult.LOAD_AND_BREAK;
}
return FieldSelectorResult.NO_LOAD;
}
@Override
public void reset() {
}
@Override
public String toString() {
return "source";
}
}

74
src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldVisitor.java Normal file
View File

@ -0,0 +1,74 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.internal;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import java.io.IOException;
/**
* An optimized field selector that loads just the _source
*/
public class SourceFieldVisitor extends BaseFieldVisitor {
public static final SourceFieldVisitor INSTANCE = new SourceFieldVisitor();
private static ThreadLocal<BytesRef> loadingContext = new ThreadLocal<BytesRef>();
private SourceFieldVisitor() {
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return loadingContext.get() != null ? Status.STOP : Status.NO;
}
@Override
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
loadingContext.set(new BytesRef(value));
}
@Override
public void reset() {
loadingContext.remove();
}
@Override
public Document createDocument() {
Document document = new Document();
document.add(new StoredField("_source", loadingContext.get().utf8ToString()));
return document;
}
public BytesRef source() {
return loadingContext.get();
}
@Override
public String toString() {
return "source";
}
}

23
src/main/java/org/elasticsearch/index/mapper/selector/AllButSourceFieldSelector.java → src/main/java/org/elasticsearch/index/mapper/selector/AllButSourceFieldVisitor.java
View File

@ -19,27 +19,24 @@
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.document.FieldSelectorResult;
import org.elasticsearch.common.lucene.document.ResetFieldSelector;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.common.lucene.document.MultipleFieldsVisitor;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import java.io.IOException;
/**
* A field selector that loads all fields except the source field.
*/
public class AllButSourceFieldSelector implements ResetFieldSelector {
public static final AllButSourceFieldSelector INSTANCE = new AllButSourceFieldSelector();
// LUCENE 4 UPGRADE: change into singleton
public class AllButSourceFieldVisitor extends MultipleFieldsVisitor {
@Override
public FieldSelectorResult accept(String fieldName) {
if (SourceFieldMapper.NAME.equals(fieldName)) {
return FieldSelectorResult.NO_LOAD;
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.NO;
}
return FieldSelectorResult.LOAD;
}
@Override
public void reset() {
return super.needsField(fieldInfo);
}
@Override

27
src/main/java/org/elasticsearch/index/mapper/selector/FieldMappersFieldSelector.java → src/main/java/org/elasticsearch/index/mapper/selector/FieldMappersFieldVisitor.java
View File

@ -19,44 +19,27 @@
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.document.FieldSelectorResult;
import org.elasticsearch.common.lucene.document.ResetFieldSelector;
import org.elasticsearch.common.lucene.document.MultipleFieldsVisitor;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMappers;
import java.util.HashSet;
/**
*
*/
public class FieldMappersFieldSelector implements ResetFieldSelector {
private final HashSet<String> names = new HashSet<String>();
public class FieldMappersFieldVisitor extends MultipleFieldsVisitor {
public void add(String fieldName) {
names.add(fieldName);
fieldsToAdd.add(fieldName);
}
public void add(FieldMappers fieldMappers) {
for (FieldMapper fieldMapper : fieldMappers) {
names.add(fieldMapper.names().indexName());
fieldsToAdd.add(fieldMapper.names().indexName());
}
}
@Override
public FieldSelectorResult accept(String fieldName) {
if (names.contains(fieldName)) {
return FieldSelectorResult.LOAD;
}
return FieldSelectorResult.NO_LOAD;
}
@Override
public void reset() {
}
@Override
public String toString() {
return "fields(" + names + ")";
return "fields(" + fieldsToAdd + ")";
}
}

60
src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldSelector.java
View File

@ -1,60 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.document.FieldSelectorResult;
import org.elasticsearch.common.lucene.document.ResetFieldSelector;
import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
/**
* An optimized field selector that loads just the uid and the routing.
*/
public class UidAndRoutingFieldSelector implements ResetFieldSelector {
private int match = 0;
@Override
public FieldSelectorResult accept(String fieldName) {
if (UidFieldMapper.NAME.equals(fieldName)) {
if (++match == 2) {
return FieldSelectorResult.LOAD_AND_BREAK;
}
return FieldSelectorResult.LOAD;
}
if (RoutingFieldMapper.NAME.equals(fieldName)) {
if (++match == 2) {
return FieldSelectorResult.LOAD_AND_BREAK;
}
return FieldSelectorResult.LOAD;
}
return FieldSelectorResult.NO_LOAD;
}
@Override
public void reset() {
match = 0;
}
@Override
public String toString() {
return "uid_and_routing";
}
}

86
src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldVisitor.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
/**
* An optimized field selector that loads just the uid and the routing.
*/
// LUCENE 4 UPGRADE: change into singleton
public class UidAndRoutingFieldVisitor extends BaseFieldVisitor {
private String uid;
private String routing;
@Override
public Document createDocument() {
Document document = new Document();
document.add(new StoredField("uid", uid));
document.add(new StoredField("_source", routing));
return document;
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
} else if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return uid != null && routing != null ? Status.STOP : Status.NO;
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) {
routing = value;
} else if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
uid = value;
}
}
@Override
public void reset() {
uid = null;
routing = null;
}
public String uid() {
return uid;
}
public String routing() {
return routing;
}
@Override
public String toString() {
return "uid_and_routing";
}
}

60
src/main/java/org/elasticsearch/index/mapper/selector/UidAndSourceFieldSelector.java
View File

@ -1,60 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.document.FieldSelectorResult;
import org.elasticsearch.common.lucene.document.ResetFieldSelector;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
/**
* An optimized field selector that loads just the uid and the source.
*/
public class UidAndSourceFieldSelector implements ResetFieldSelector {
private int match = 0;
@Override
public FieldSelectorResult accept(String fieldName) {
if (UidFieldMapper.NAME.equals(fieldName)) {
if (++match == 2) {
return FieldSelectorResult.LOAD_AND_BREAK;
}
return FieldSelectorResult.LOAD;
}
if (SourceFieldMapper.NAME.equals(fieldName)) {
if (++match == 2) {
return FieldSelectorResult.LOAD_AND_BREAK;
}
return FieldSelectorResult.LOAD;
}
return FieldSelectorResult.NO_LOAD;
}
@Override
public void reset() {
match = 0;
}
@Override
public String toString() {
return "uid_and_source";
}
}

80
src/main/java/org/elasticsearch/index/mapper/selector/UidAndSourceFieldVisitor.java Normal file
View File

@ -0,0 +1,80 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
/**
* An optimized field selector that loads just the uid and the source.
*/
// LUCENE 4 UPGRADE: change into singleton
public class UidAndSourceFieldVisitor extends BaseFieldVisitor {
private String uid;
private BytesRef source;
@Override
public Document createDocument() {
Document document = new Document();
document.add(new StoredField("uid", uid));
document.add(new StoredField("_source", source));
return document;
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
} else if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return uid != null && source != null ? Status.STOP : Status.NO;
}
@Override
public void reset() {
uid = null;
source = null;
}
@Override
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
source = new BytesRef(value);
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
uid = value;
}
@Override
public String toString() {
return "uid_and_source";
}
}

41
src/main/java/org/elasticsearch/index/mapper/selector/UidFieldSelector.java → src/main/java/org/elasticsearch/index/mapper/selector/UidFieldVisitor.java
View File

@ -19,31 +19,52 @@
package org.elasticsearch.index.mapper.selector;
import org.apache.lucene.document.FieldSelectorResult;
import org.elasticsearch.common.lucene.document.ResetFieldSelector;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
/**
* An optimized field selector that loads just the uid.
*/
public class UidFieldSelector implements ResetFieldSelector {
public class UidFieldVisitor extends BaseFieldVisitor {
public static final UidFieldSelector INSTANCE = new UidFieldSelector();
private UidFieldSelector() {
public static final UidFieldVisitor INSTANCE = new UidFieldVisitor();
private static ThreadLocal<String> loadingContext = new ThreadLocal<String>();
private UidFieldVisitor() {
}
@Override
public FieldSelectorResult accept(String fieldName) {
if (UidFieldMapper.NAME.equals(fieldName)) {
return FieldSelectorResult.LOAD_AND_BREAK;
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
loadingContext.set(value);
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return FieldSelectorResult.NO_LOAD;
return loadingContext.get() != null ? Status.STOP : Status.NO;
}
@Override
public void reset() {
loadingContext.remove();
}
@Override
public Document createDocument() {
Document document = new Document();
document.add(new StoredField("_uid", loadingContext.get()));
return document;
}
public String uid() {
return loadingContext.get();
}
@Override

24
src/main/java/org/elasticsearch/index/percolator/PercolatorService.java
View File

@ -21,9 +21,11 @@ package org.elasticsearch.index.percolator;
import com.google.common.collect.Maps;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.TermFilter;
@ -36,7 +38,7 @@ import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.mapper.selector.UidAndSourceFieldSelector;
import org.elasticsearch.index.mapper.selector.UidAndSourceFieldVisitor;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.IndexShardState;
@ -135,7 +137,7 @@ public class PercolatorService extends AbstractIndexComponent {
}
private Filter indexQueriesFilter(String indexName) {
return percolatorIndexService().cache().filter().cache(new TermFilter(TypeFieldMapper.TERM_FACTORY.createTerm(indexName)));
return percolatorIndexService().cache().filter().cache(new TermFilter(new Term(TypeFieldMapper.NAME, indexName)));
}
private boolean percolatorAllocated() {
@ -157,7 +159,7 @@ public class PercolatorService extends AbstractIndexComponent {
class QueriesLoaderCollector extends Collector {
private IndexReader reader;
private AtomicReader reader;
private Map<String, Query> queries = Maps.newHashMap();
@ -172,19 +174,21 @@ public class PercolatorService extends AbstractIndexComponent {
@Override
public void collect(int doc) throws IOException {
// the _source is the query
Document document = reader.document(doc, new UidAndSourceFieldSelector());
UidAndSourceFieldVisitor fieldVisitor = new UidAndSourceFieldVisitor();
reader.document(doc, fieldVisitor);
Document document = fieldVisitor.createDocument();
String id = Uid.createUid(document.get(UidFieldMapper.NAME)).id();
try {
Fieldable sourceField = document.getFieldable(SourceFieldMapper.NAME);
queries.put(id, percolator.parseQuery(id, new BytesArray(sourceField.getBinaryValue(), sourceField.getBinaryOffset(), sourceField.getBinaryLength())));
BytesRef sourceVal = document.getBinaryValue(SourceFieldMapper.NAME);
queries.put(id, percolator.parseQuery(id, new BytesArray(sourceVal.bytes, sourceVal.offset, sourceVal.length)));
} catch (Exception e) {
logger.warn("failed to add query [{}]", e, id);
}
}
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
this.reader = reader;
public void setNextReader(AtomicReaderContext context) throws IOException {
this.reader = context.reader();
}
@Override

23
src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
View File

@ -19,8 +19,8 @@
package org.elasticsearch.indices.ttl;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
@ -44,16 +44,16 @@ import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMappers;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.mapper.selector.UidAndRoutingFieldSelector;
import org.elasticsearch.index.mapper.selector.UidAndRoutingFieldVisitor;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.shard.IndexShardState;
import org.elasticsearch.index.shard.service.IndexShard;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.node.settings.NodeSettingsService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@ -220,7 +220,7 @@ public class IndicesTTLService extends AbstractLifecycleComponent<IndicesTTLServ
}
private class ExpiredDocsCollector extends Collector {
private IndexReader indexReader;
private AtomicReaderContext context;
private List<DocToPurge> docsToPurge = new ArrayList<DocToPurge>();
public ExpiredDocsCollector() {
@ -235,17 +235,18 @@ public class IndicesTTLService extends AbstractLifecycleComponent<IndicesTTLServ
public void collect(int doc) {
try {
Document document = indexReader.document(doc, new UidAndRoutingFieldSelector());
String uid = document.getFieldable(UidFieldMapper.NAME).stringValue();
long version = UidField.loadVersion(indexReader, UidFieldMapper.TERM_FACTORY.createTerm(uid));
docsToPurge.add(new DocToPurge(Uid.typeFromUid(uid), Uid.idFromUid(uid), version, document.get(RoutingFieldMapper.NAME)));
UidAndRoutingFieldVisitor fieldVisitor = new UidAndRoutingFieldVisitor();
context.reader().document(doc, new UidAndRoutingFieldVisitor());
String uid = fieldVisitor.uid();
long version = UidField.loadVersion(context, new Term(UidFieldMapper.NAME, uid));
docsToPurge.add(new DocToPurge(Uid.typeFromUid(uid), Uid.idFromUid(uid), version, fieldVisitor.routing()));
} catch (Exception e) {
logger.trace("failed to collect doc", e);
}
}
public void setNextReader(IndexReader reader, int docBase) {
this.indexReader = reader;
public void setNextReader(AtomicReaderContext context) throws IOException {
this.context = context;
}
public List<DocToPurge> getDocsToPurge() {

95
src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
View File

@ -22,12 +22,14 @@ package org.elasticsearch.search.fetch;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.ReaderUtil;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.document.ResetFieldSelector;
import org.elasticsearch.common.lucene.document.BaseFieldVisitor;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
@ -35,10 +37,10 @@ import org.elasticsearch.index.mapper.FieldMappers;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.mapper.selector.AllButSourceFieldSelector;
import org.elasticsearch.index.mapper.selector.FieldMappersFieldSelector;
import org.elasticsearch.index.mapper.selector.UidAndSourceFieldSelector;
import org.elasticsearch.index.mapper.selector.UidFieldSelector;
import org.elasticsearch.index.mapper.selector.AllButSourceFieldVisitor;
import org.elasticsearch.index.mapper.selector.FieldMappersFieldVisitor;
import org.elasticsearch.index.mapper.selector.UidAndSourceFieldVisitor;
import org.elasticsearch.index.mapper.selector.UidFieldVisitor;
import org.elasticsearch.indices.TypeMissingException;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchParseElement;
@ -88,28 +90,28 @@ public class FetchPhase implements SearchPhase {
}
public void execute(SearchContext context) {
ResetFieldSelector fieldSelector;
BaseFieldVisitor fieldVisitor;
List<String> extractFieldNames = null;
boolean sourceRequested = false;
if (!context.hasFieldNames()) {
if (context.hasPartialFields()) {
// partial fields need the source, so fetch it, but don't return it
fieldSelector = new UidAndSourceFieldSelector();
fieldVisitor = new UidAndSourceFieldVisitor();
sourceRequested = false;
} else if (context.hasScriptFields()) {
// we ask for script fields, and no field names, don't load the source
fieldSelector = UidFieldSelector.INSTANCE;
fieldVisitor = UidFieldVisitor.INSTANCE;
sourceRequested = false;
} else {
fieldSelector = new UidAndSourceFieldSelector();
fieldVisitor = new UidAndSourceFieldVisitor();
sourceRequested = true;
}
} else if (context.fieldNames().isEmpty()) {
fieldSelector = UidFieldSelector.INSTANCE;
fieldVisitor = UidFieldVisitor.INSTANCE;
sourceRequested = false;
} else {
boolean loadAllStored = false;
FieldMappersFieldSelector fieldSelectorMapper = null;
FieldMappersFieldVisitor fieldVisitorMapper = null;
for (String fieldName : context.fieldNames()) {
if (fieldName.equals("*")) {
loadAllStored = true;
@ -121,10 +123,10 @@ public class FetchPhase implements SearchPhase {
}
FieldMappers x = context.smartNameFieldMappers(fieldName);
if (x != null && x.mapper().stored()) {
if (fieldSelectorMapper == null) {
fieldSelectorMapper = new FieldMappersFieldSelector();
if (fieldVisitorMapper == null) {
fieldVisitorMapper = new FieldMappersFieldVisitor();
}
fieldSelectorMapper.add(x);
fieldVisitorMapper.add(x);
} else {
if (extractFieldNames == null) {
extractFieldNames = Lists.newArrayList();
@ -135,29 +137,29 @@ public class FetchPhase implements SearchPhase {
if (loadAllStored) {
if (sourceRequested || extractFieldNames != null) {
fieldSelector = null; // load everything, including _source
fieldVisitor = null; // load everything, including _source
} else {
fieldSelector = AllButSourceFieldSelector.INSTANCE;
fieldVisitor = new AllButSourceFieldVisitor();
}
} else if (fieldSelectorMapper != null) {
} else if (fieldVisitorMapper != null) {
// we are asking specific stored fields, just add the UID and be done
fieldSelectorMapper.add(UidFieldMapper.NAME);
fieldVisitorMapper.add(UidFieldMapper.NAME);
if (extractFieldNames != null || sourceRequested) {
fieldSelectorMapper.add(SourceFieldMapper.NAME);
fieldVisitorMapper.add(SourceFieldMapper.NAME);
}
fieldSelector = fieldSelectorMapper;
fieldVisitor = fieldVisitorMapper;
} else if (extractFieldNames != null || sourceRequested) {
fieldSelector = new UidAndSourceFieldSelector();
fieldVisitor = new UidAndSourceFieldVisitor();
} else {
fieldSelector = UidFieldSelector.INSTANCE;
fieldVisitor = UidFieldVisitor.INSTANCE;
}
}
InternalSearchHit[] hits = new InternalSearchHit[context.docIdsToLoadSize()];
for (int index = 0; index < context.docIdsToLoadSize(); index++) {
int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index];
Document doc = loadDocument(context, fieldSelector, docId);
Uid uid = extractUid(context, doc, fieldSelector);
Document doc = loadDocument(context, fieldVisitor, docId);
Uid uid = extractUid(context, doc, fieldVisitor);
DocumentMapper documentMapper = context.mapperService().documentMapper(uid.type());
@ -173,7 +175,7 @@ public class FetchPhase implements SearchPhase {
hits[index] = searchHit;
for (Object oField : doc.getFields()) {
Fieldable field = (Fieldable) oField;
IndexableField field = (IndexableField) oField;
String name = field.name();
// ignore UID, we handled it above
@ -192,12 +194,13 @@ public class FetchPhase implements SearchPhase {
FieldMapper mapper = fieldMappers.mapper();
if (mapper != null) {
name = mapper.names().fullName();
value = mapper.valueForSearch(field);
// LUCENE 4 UPGRADE: do we really need to use Field instead of IndexableField?
value = mapper.valueForSearch((Field) field);
}
}
if (value == null) {
if (field.isBinary()) {
value = new BytesArray(field.getBinaryValue(), field.getBinaryOffset(), field.getBinaryLength());
if (field.binaryValue() != null) {
value = new BytesArray(field.binaryValue());
} else {
value = field.stringValue();
}
@ -215,12 +218,12 @@ public class FetchPhase implements SearchPhase {
hitField.values().add(value);
}
int readerIndex = context.searcher().readerIndex(docId);
IndexReader subReader = context.searcher().subReaders()[readerIndex];
int subDoc = docId - context.searcher().docStarts()[readerIndex];
int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves());
AtomicReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex);
int subDoc = docId - subReaderContext.docBase;
// go over and extract fields that are not mapped / stored
context.lookup().setNextReader(subReader);
context.lookup().setNextReader(subReaderContext);
context.lookup().setNextDocId(subDoc);
if (source != null) {
context.lookup().source().setNextSource(new BytesArray(source));
@ -246,7 +249,7 @@ public class FetchPhase implements SearchPhase {
for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
if (fetchSubPhase.hitExecutionNeeded(context)) {
hitContext.reset(searchHit, subReader, subDoc, context.searcher().getIndexReader(), docId, doc);
hitContext.reset(searchHit, subReaderContext, subDoc, context.searcher().getIndexReader(), docId, doc);
fetchSubPhase.hitExecute(context, hitContext);
}
}
@ -262,30 +265,36 @@ public class FetchPhase implements SearchPhase {
}
private byte[] extractSource(Document doc, DocumentMapper documentMapper) {
Fieldable sourceField = doc.getFieldable(SourceFieldMapper.NAME);
IndexableField sourceField = doc.getField(SourceFieldMapper.NAME);
if (sourceField != null) {
return documentMapper.sourceMapper().nativeValue(sourceField);
//LUCENE 4 UPGRADE: I think all sourceFields are of type Field
return documentMapper.sourceMapper().nativeValue((Field) sourceField);
}
return null;
}
private Uid extractUid(SearchContext context, Document doc, @Nullable ResetFieldSelector fieldSelector) {
private Uid extractUid(SearchContext context, Document doc, @Nullable BaseFieldVisitor fieldVisitor) {
String sUid = doc.get(UidFieldMapper.NAME);
if (sUid != null) {
return Uid.createUid(sUid);
}
// no type, nothing to do (should not really happen)
List<String> fieldNames = new ArrayList<String>();
for (Fieldable field : doc.getFields()) {
for (IndexableField field : doc.getFields()) {
fieldNames.add(field.name());
}
throw new FetchPhaseExecutionException(context, "Failed to load uid from the index, missing internal _uid field, current fields in the doc [" + fieldNames + "], selector [" + fieldSelector + "]");
throw new FetchPhaseExecutionException(context, "Failed to load uid from the index, missing internal _uid field, current fields in the doc [" + fieldNames + "], selector [" + fieldVisitor + "]");
}
private Document loadDocument(SearchContext context, @Nullable ResetFieldSelector fieldSelector, int docId) {
private Document loadDocument(SearchContext context, @Nullable BaseFieldVisitor fieldVisitor, int docId) {
try {
if (fieldSelector != null) fieldSelector.reset();
return context.searcher().doc(docId, fieldSelector);
if (fieldVisitor != null) {
fieldVisitor.reset();
} else {
return context.searcher().doc(docId);
}
context.searcher().doc(docId, fieldVisitor);
return fieldVisitor.createDocument();
} catch (IOException e) {
throw new FetchPhaseExecutionException(context, "Failed to fetch doc id [" + docId + "]", e);
}

16
src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
View File

@ -21,6 +21,8 @@ package org.elasticsearch.search.fetch;
import com.google.common.collect.Maps;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.search.SearchParseElement;
@ -38,14 +40,14 @@ public interface FetchSubPhase {
private InternalSearchHit hit;
private IndexReader topLevelReader;
private int topLevelDocId;
private IndexReader reader;
private AtomicReaderContext readerContext;
private int docId;
private Document doc;
private Map<String, Object> cache;
public void reset(InternalSearchHit hit, IndexReader reader, int docId, IndexReader topLevelReader, int topLevelDocId, Document doc) {
public void reset(InternalSearchHit hit, AtomicReaderContext context, int docId, IndexReader topLevelReader, int topLevelDocId, Document doc) {
this.hit = hit;
this.reader = reader;
this.readerContext = context;
this.docId = docId;
this.topLevelReader = topLevelReader;
this.topLevelDocId = topLevelDocId;
@ -56,8 +58,12 @@ public interface FetchSubPhase {
return hit;
}
public IndexReader reader() {
return reader;
public AtomicReader reader() {
return readerContext.reader();
}
public AtomicReaderContext readerContext() {
return readerContext;
}
public int docId() {

23
src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
View File

@ -23,9 +23,6 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.highlight.*;
import org.apache.lucene.search.highlight.Formatter;
@ -34,7 +31,7 @@ import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FastStringReader;
import org.elasticsearch.common.lucene.document.SingleFieldSelector;
import org.elasticsearch.common.lucene.document.SingleFieldVisitor;
import org.elasticsearch.common.lucene.search.vectorhighlight.SimpleBoundaryScanner2;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.StringText;
@ -129,9 +126,9 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
if (field.highlighterType() == null) {
// if we can do highlighting using Term Vectors, use FastVectorHighlighter, otherwise, use the
// slower plain highlighter
useFastVectorHighlighter = mapper.termVector() == Field.TermVector.WITH_POSITIONS_OFFSETS;
useFastVectorHighlighter = mapper.storeTermVectors() && mapper.storeTermVectorOffsets() && mapper.storeTermVectorPositions();
} else if (field.highlighterType().equals("fast-vector-highlighter") || field.highlighterType().equals("fvh")) {
if (mapper.termVector() != Field.TermVector.WITH_POSITIONS_OFFSETS) {
if (!(mapper.storeTermVectors() && mapper.storeTermVectorOffsets() && mapper.storeTermVectorPositions())) {
throw new FetchPhaseExecutionException(context, "the field [" + field.field() + "] should be indexed with term vector with position offsets to be used with fast vector highlighter");
}
useFastVectorHighlighter = true;
@ -170,19 +167,15 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
List<Object> textsToHighlight;
if (mapper.stored()) {
try {
Document doc = hitContext.reader().document(hitContext.docId(), new SingleFieldSelector(mapper.names().indexName()));
textsToHighlight = new ArrayList<Object>(doc.getFields().size());
for (Fieldable docField : doc.getFields()) {
if (docField.stringValue() != null) {
textsToHighlight.add(docField.stringValue());
}
}
SingleFieldVisitor fieldVisitor = new SingleFieldVisitor(mapper.names().indexName());
hitContext.reader().document(hitContext.docId(), fieldVisitor);
textsToHighlight = (List<Object>) fieldVisitor.values();
} catch (Exception e) {
throw new FetchPhaseExecutionException(context, "Failed to highlight field [" + field.field() + "]", e);
}
} else {
SearchLookup lookup = context.lookup();
lookup.setNextReader(hitContext.reader());
lookup.setNextReader(hitContext.readerContext());
lookup.setNextDocId(hitContext.docId());
textsToHighlight = lookup.source().extractRawValues(mapper.names().sourcePath());
}
@ -193,7 +186,7 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
try {
for (Object textToHighlight : textsToHighlight) {
String text = textToHighlight.toString();
Analyzer analyzer = context.mapperService().documentMapper(hitContext.hit().type()).mappers().indexAnalyzer();
Analyzer analyzer = context.mapperService().documentMapper(hitContext.hit().type()).mappers().indexAnalyzer()
TokenStream tokenStream = analyzer.reusableTokenStream(mapper.names().indexName(), new FastStringReader(text));
TextFragment[] bestTextFragments = entry.highlighter.getBestTextFragments(tokenStream, text, false, numberOfFragments);
for (TextFragment bestTextFragment : bestTextFragments) {

12
src/main/java/org/elasticsearch/search/lookup/FieldsLookup.java
View File

@ -25,7 +25,7 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.lucene.document.SingleFieldSelector;
import org.elasticsearch.common.lucene.document.SingleFieldVisitor;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
@ -51,7 +51,7 @@ public class FieldsLookup implements Map {
private final Map<String, FieldLookup> cachedFieldData = Maps.newHashMap();
private final SingleFieldSelector fieldSelector = new SingleFieldSelector();
private final SingleFieldVisitor fieldVisitor = new SingleFieldVisitor();
FieldsLookup(MapperService mapperService, @Nullable String[] types) {
this.mapperService = mapperService;
@ -152,11 +152,15 @@ public class FieldsLookup implements Map {
cachedFieldData.put(name, data);
}
if (data.doc() == null) {
fieldSelector.name(data.mapper().names().indexName());
fieldVisitor.name(data.mapper().names().indexName());
try {
data.doc(reader.document(docId, fieldSelector));
reader.document(docId, fieldVisitor);
// LUCENE 4 UPGRADE: Only one field we don't need document
data.doc(fieldVisitor.createDocument());
} catch (IOException e) {
throw new ElasticSearchParseException("failed to load field [" + name + "]", e);
} finally {
fieldVisitor.reset();
}
}
return data;

17
src/main/java/org/elasticsearch/search/lookup/SourceLookup.java
View File

@ -20,18 +20,14 @@
package org.elasticsearch.search.lookup;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.SourceFieldSelector;
import org.elasticsearch.index.mapper.internal.SourceFieldVisitor;
import java.util.Collection;
import java.util.List;
@ -64,16 +60,17 @@ public class SourceLookup implements Map {
return source;
}
try {
Document doc = reader.document(docId, SourceFieldSelector.INSTANCE);
IndexableField sourceField = doc.getField(SourceFieldMapper.NAME);
if (sourceField == null) {
source = ImmutableMap.of();
reader.document(docId, SourceFieldVisitor.INSTANCE);
BytesRef source = SourceFieldVisitor.INSTANCE.source();
if (source == null) {
this.source = ImmutableMap.of();
} else {
BytesRef source = sourceField.binaryValue();
this.source = sourceAsMap(source.bytes, source.offset, source.length);
}
} catch (Exception e) {
throw new ElasticSearchParseException("failed to parse / load source", e);
} finally {
SourceFieldVisitor.INSTANCE.reset();
}
return this.source;
}