honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

move scripts to use new field data

This commit is contained in:
Shay Banon 2013-01-22 13:45:00 +01:00
parent 772ee9db54
commit be1e5becbb
13 changed files with 145 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
View File

@ -20,8 +20,11 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.index.fielddata.util.*;
import org.elasticsearch.index.mapper.geo.GeoPoint;
import org.elasticsearch.index.search.geo.GeoDistance;
import org.joda.time.MutableDateTime;
/**
* Script level doc values, the assumption is that any implementation will implement a <code>getValue</code>
@ -176,6 +179,7 @@ public interface ScriptDocValues {
static class NumericLong implements ScriptDocValues {
private final LongValues values;
private final MutableDateTime date = new MutableDateTime(0);
private int docId;
public NumericLong(LongValues values) {
@ -196,6 +200,11 @@ public interface ScriptDocValues {
return values.getValue(docId);
}
public MutableDateTime getDate() {
date.setMillis(getValue());
return date;
}
public LongArrayRef getValues() {
return values.getValues(docId);
}
@ -283,5 +292,80 @@ public interface ScriptDocValues {
public GeoPointArrayRef getValues() {
return values.getValues(docId);
}
public double factorDistance(double lat, double lon) {
GeoPoint point = getValue();
return GeoDistance.FACTOR.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
}
public double factorDistanceWithDefault(double lat, double lon, double defaultValue) {
if (isEmpty()) {
return defaultValue;
}
GeoPoint point = getValue();
return GeoDistance.FACTOR.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
}
public double factorDistance02(double lat, double lon) {
GeoPoint point = getValue();
return GeoDistance.FACTOR.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES) + 1;
}
public double factorDistance13(double lat, double lon) {
GeoPoint point = getValue();
return GeoDistance.FACTOR.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES) + 2;
}
public double arcDistance(double lat, double lon) {
GeoPoint point = getValue();
return GeoDistance.ARC.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
}
public double arcDistanceWithDefault(double lat, double lon, double defaultValue) {
if (isEmpty()) {
return defaultValue;
}
GeoPoint point = getValue();
return GeoDistance.ARC.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
}
public double arcDistanceInKm(double lat, double lon) {
GeoPoint point = getValue();
return GeoDistance.ARC.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.KILOMETERS);
}
public double arcDistanceInKmWithDefault(double lat, double lon, double defaultValue) {
if (isEmpty()) {
return defaultValue;
}
GeoPoint point = getValue();
return GeoDistance.ARC.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.KILOMETERS);
}
public double distance(double lat, double lon) {
GeoPoint point = getValue();
return GeoDistance.PLANE.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
}
public double distanceWithDefault(double lat, double lon, double defaultValue) {
if (isEmpty()) {
return defaultValue;
}
GeoPoint point = getValue();
return GeoDistance.PLANE.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
}
public double distanceInKm(double lat, double lon) {
GeoPoint point = getValue();
return GeoDistance.PLANE.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.KILOMETERS);
}
public double distanceInKmWithDefault(double lat, double lon, double defaultValue) {
if (isEmpty()) {
return defaultValue;
}
GeoPoint point = getValue();
return GeoDistance.PLANE.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.KILOMETERS);
}
}
}

6
src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
View File

@ -91,7 +91,7 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericByte(getByteValues());
}
@Override
@ -458,7 +458,7 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericByte(getByteValues());
}
@Override
@ -733,7 +733,7 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericByte(getByteValues());
}
@Override

11
src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
View File

@ -23,7 +23,10 @@ import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.common.RamUsage;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.util.*;
import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
import org.elasticsearch.index.fielddata.util.FloatArrayRef;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
import org.elasticsearch.index.fielddata.util.LongArrayRef;
/**
*/
@ -88,7 +91,7 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericFloat(getFloatValues());
}
@Override
@ -455,7 +458,7 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericFloat(getFloatValues());
}
@Override
@ -730,7 +733,7 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericFloat(getFloatValues());
}
@Override

4
src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
View File

@ -457,7 +457,7 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericInteger(getIntValues());
}
@Override
@ -731,7 +731,7 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericInteger(getIntValues());
}
@Override

6
src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
View File

@ -91,7 +91,7 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericLong(getLongValues());
}
@Override
@ -452,7 +452,7 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericLong(getLongValues());
}
@Override
@ -723,7 +723,7 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericLong(getLongValues());
}
@Override

6
src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
View File

@ -91,7 +91,7 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericShort(getShortValues());
}
@Override
@ -458,7 +458,7 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericShort(getShortValues());
}
@Override
@ -733,7 +733,7 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
return new ScriptDocValues.NumericShort(getShortValues());
}
@Override

16
src/main/java/org/elasticsearch/index/get/ShardGetService.java
View File

@ -28,8 +28,8 @@ import org.elasticsearch.common.lucene.uid.UidField;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor;
@ -62,7 +62,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
private final MapperService mapperService;
private final IndexCache indexCache;
private final IndexFieldDataService fieldDataService;
private IndexShard indexShard;
@ -72,11 +72,11 @@ public class ShardGetService extends AbstractIndexShardComponent {
@Inject
public ShardGetService(ShardId shardId, @IndexSettings Settings indexSettings, ScriptService scriptService,
MapperService mapperService, IndexCache indexCache) {
MapperService mapperService, IndexFieldDataService fieldDataService) {
super(shardId, indexSettings);
this.scriptService = scriptService;
this.mapperService = mapperService;
this.indexCache = indexCache;
this.fieldDataService = fieldDataService;
}
public GetStats stats() {
@ -213,7 +213,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
} else {
if (field.contains("_source.")) {
if (searchLookup == null) {
searchLookup = new SearchLookup(mapperService, indexCache.fieldData(), new String[]{type});
searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
}
if (sourceAsMap == null) {
sourceAsMap = SourceLookup.sourceAsMap(source.source);
@ -236,7 +236,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
}
} else {
if (searchLookup == null) {
searchLookup = new SearchLookup(mapperService, indexCache.fieldData(), new String[]{type});
searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
searchLookup.source().setNextSource(source.source);
}
@ -306,7 +306,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
Object value = null;
if (field.contains("_source.") || field.contains("doc[")) {
if (searchLookup == null) {
searchLookup = new SearchLookup(mapperService, indexCache.fieldData(), new String[]{type});
searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
}
SearchScript searchScript = scriptService.search(searchLookup, "mvel", field, null);
searchScript.setNextReader(docIdAndVersion.reader);
@ -324,7 +324,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
FieldMappers x = docMapper.mappers().smartName(field);
if (x == null || !x.mapper().fieldType().stored()) {
if (searchLookup == null) {
searchLookup = new SearchLookup(mapperService, indexCache.fieldData(), new String[]{type});
searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
searchLookup.setNextReader(docIdAndVersion.reader);
searchLookup.setNextDocId(docIdAndVersion.docId);
}

6
src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.engine.IndexEngine;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.index.settings.IndexSettings;
@ -77,6 +78,8 @@ public class IndexQueryParserService extends AbstractIndexComponent {
final IndexCache indexCache;
final IndexFieldDataService fieldDataService;
final IndexEngine indexEngine;
private final Map<String, QueryParser> queryParsers;
@ -90,7 +93,7 @@ public class IndexQueryParserService extends AbstractIndexComponent {
public IndexQueryParserService(Index index, @IndexSettings Settings indexSettings,
IndicesQueriesRegistry indicesQueriesRegistry,
ScriptService scriptService, AnalysisService analysisService,
MapperService mapperService, IndexCache indexCache, IndexEngine indexEngine,
MapperService mapperService, IndexCache indexCache, IndexFieldDataService fieldDataService, IndexEngine indexEngine,
@Nullable SimilarityService similarityService,
@Nullable Map<String, QueryParserFactory> namedQueryParsers,
@Nullable Map<String, FilterParserFactory> namedFilterParsers) {
@ -100,6 +103,7 @@ public class IndexQueryParserService extends AbstractIndexComponent {
this.mapperService = mapperService;
this.similarityService = similarityService;
this.indexCache = indexCache;
this.fieldDataService = fieldDataService;
this.indexEngine = indexEngine;
this.defaultField = indexSettings.get("index.query.default_field", AllFieldMapper.NAME);

8
src/main/java/org/elasticsearch/index/query/QueryParseContext.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.index.query;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.lucene.queryparser.classic.MapperQueryParser;
import org.apache.lucene.queryparser.classic.QueryParserSettings;
import org.apache.lucene.search.Filter;
@ -34,6 +33,7 @@ import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
import org.elasticsearch.index.engine.IndexEngine;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMappers;
import org.elasticsearch.index.mapper.MapperService;
@ -130,6 +130,10 @@ public class QueryParseContext {
return indexQueryParser.indexCache;
}
public IndexFieldDataService fieldData() {
return indexQueryParser.fieldDataService;
}
public String defaultField() {
return indexQueryParser.defaultField();
}
@ -291,7 +295,7 @@ public class QueryParseContext {
return current.lookup();
}
if (lookup == null) {
lookup = new SearchLookup(mapperService(), indexCache().fieldData(), null);
lookup = new SearchLookup(mapperService(), fieldData(), null);
}
return lookup;
}

6
src/main/java/org/elasticsearch/script/ScriptService.java
View File

@ -32,7 +32,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.cache.field.data.FieldDataCache;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.script.mvel.MvelScriptEngineService;
import org.elasticsearch.search.lookup.SearchLookup;
@ -193,8 +193,8 @@ public class ScriptService extends AbstractComponent {
return search(compile(lang, script), lookup, vars);
}
public SearchScript search(MapperService mapperService, FieldDataCache fieldDataCache, String lang, String script, @Nullable Map<String, Object> vars) {
return search(compile(lang, script), new SearchLookup(mapperService, fieldDataCache, null), vars);
public SearchScript search(MapperService mapperService, IndexFieldDataService fieldDataService, String lang, String script, @Nullable Map<String, Object> vars) {
return search(compile(lang, script), new SearchLookup(mapperService, fieldDataService, null), vars);
}
public Object execute(CompiledScript compiledScript, Map vars) {

2
src/main/java/org/elasticsearch/search/internal/SearchContext.java
View File

@ -546,7 +546,7 @@ public class SearchContext implements Releasable {
public SearchLookup lookup() {
// TODO: The types should take into account the parsing context in QueryParserContext...
if (searchLookup == null) {
searchLookup = new SearchLookup(mapperService(), fieldDataCache(), request.types());
searchLookup = new SearchLookup(mapperService(), fieldData(), request.types());
}
return searchLookup;
}

44
src/main/java/org/elasticsearch/search/lookup/DocLookup.java
View File

@ -20,16 +20,14 @@
package org.elasticsearch.search.lookup;
import com.google.common.collect.Maps;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Scorer;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.cache.field.data.FieldDataCache;
import org.elasticsearch.index.field.data.DocFieldData;
import org.elasticsearch.index.field.data.FieldData;
import org.elasticsearch.index.field.data.NumericDocFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
@ -44,24 +42,23 @@ import java.util.Set;
*/
public class DocLookup implements Map {
private final Map<String, FieldData> localCacheFieldData = Maps.newHashMapWithExpectedSize(4);
private final Map<String, ScriptDocValues> localCacheFieldData = Maps.newHashMapWithExpectedSize(4);
private final MapperService mapperService;
private final FieldDataCache fieldDataCache;
private final IndexFieldDataService fieldDataService;
@Nullable
private final String[] types;
private AtomicReader reader;
private AtomicReaderContext reader;
private Scorer scorer;
private int docId = -1;
DocLookup(MapperService mapperService, FieldDataCache fieldDataCache, @Nullable String[] types) {
DocLookup(MapperService mapperService, IndexFieldDataService fieldDataService, @Nullable String[] types) {
this.mapperService = mapperService;
this.fieldDataCache = fieldDataCache;
this.fieldDataService = fieldDataService;
this.types = types;
}
@ -69,15 +66,11 @@ public class DocLookup implements Map {
return this.mapperService;
}
public FieldDataCache fieldDataCache() {
return this.fieldDataCache;
}
public void setNextReader(AtomicReaderContext context) {
if (this.reader == context.reader()) { // if we are called with the same reader, don't invalidate source
if (this.reader == context) { // if we are called with the same reader, don't invalidate source
return;
}
this.reader = context.reader();
this.reader = context;
this.docId = -1;
localCacheFieldData.clear();
}
@ -110,27 +103,24 @@ public class DocLookup implements Map {
public Object get(Object key) {
// assume its a string...
String fieldName = key.toString();
FieldData fieldData = localCacheFieldData.get(fieldName);
if (fieldData == null) {
ScriptDocValues scriptValues = localCacheFieldData.get(fieldName);
if (scriptValues == null) {
FieldMapper mapper = mapperService.smartNameFieldMapper(fieldName, types);
if (mapper == null) {
throw new ElasticSearchIllegalArgumentException("No field found for [" + fieldName + "] in mapping with types " + Arrays.toString(types) + "");
}
try {
fieldData = fieldDataCache.cache(mapper.fieldDataType(), reader, mapper.names().indexName());
} catch (IOException e) {
throw new ElasticSearchException("Failed to load field data for [" + fieldName + "]", e);
}
localCacheFieldData.put(fieldName, fieldData);
scriptValues = fieldDataService.getForField(mapper).load(reader).getScriptValues();
localCacheFieldData.put(fieldName, scriptValues);
}
return fieldData.docFieldData(docId);
scriptValues.setNextDocId(docId);
return scriptValues;
}
public boolean containsKey(Object key) {
// assume its a string...
String fieldName = key.toString();
FieldData fieldData = localCacheFieldData.get(fieldName);
if (fieldData == null) {
ScriptDocValues scriptValues = localCacheFieldData.get(fieldName);
if (scriptValues == null) {
FieldMapper mapper = mapperService.smartNameFieldMapper(fieldName, types);
if (mapper == null) {
return false;

6
src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
View File

@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableMap;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Scorer;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.cache.field.data.FieldDataCache;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
/**
@ -39,8 +39,8 @@ public class SearchLookup {
final ImmutableMap<String, Object> asMap;
public SearchLookup(MapperService mapperService, FieldDataCache fieldDataCache, @Nullable String[] types) {
docMap = new DocLookup(mapperService, fieldDataCache, types);
public SearchLookup(MapperService mapperService, IndexFieldDataService fieldDataService, @Nullable String[] types) {
docMap = new DocLookup(mapperService, fieldDataService, types);
sourceLookup = new SourceLookup();
fieldsLookup = new FieldsLookup(mapperService, types);
asMap = ImmutableMap.<String, Object>of("doc", docMap, "_doc", docMap, "_source", sourceLookup, "_fields", fieldsLookup);