honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve numeric stream analysis performance

This commit is contained in:
kimchy 2011-01-09 19:35:24 +02:00
parent 3c3d01347e
commit 5c08271e74
10 changed files with 126 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.idea/runConfigurations/Bootstrap__No_Plugins_.xml
View File

@ -7,7 +7,7 @@
</pattern>
</extension>
<option name="MAIN_CLASS_NAME" value="org.elasticsearch.bootstrap.Bootstrap" />
<option name="VM_PARAMETERS" value="-server -Xmx1g -Des-foreground=yes -XX:+AggressiveOpts -XX:+UseParNewGC -XX:+UseConcMarkSweepGCa" />
<option name="VM_PARAMETERS" value="-server -Xmx1g -Des-foreground=yes -XX:+AggressiveOpts -XX:+UseParNewGC -XX:+UseConcMarkSweepGC" />
<option name="PROGRAM_PARAMETERS" value="" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />

11
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/BoostFieldMapper.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.index.mapper.xcontent;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.Filter;
@ -152,15 +151,7 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements org.el
return null;
}
context.doc().setBoost(value);
return new CustomNumericField(names.indexName(), indexed(), stored() ? Numbers.floatToBytes(value) : null) {
@Override public TokenStream tokenStreamValue() {
if (indexed()) {
return popCachedStream(precisionStep).setFloatValue(value);
} else {
return null;
}
}
};
return new FloatFieldMapper.CustomFloatNumericField(this, value);
}
private float parseFloatValue(ParseContext context) throws IOException {

11
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/DateFieldMapper.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.index.mapper.xcontent;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.Filter;
@ -200,15 +199,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
}
final long value = parseStringValue(dateAsString);
return new CustomNumericField(names.indexName(), indexed(), stored() ? Numbers.longToBytes(value) : null) {
@Override public TokenStream tokenStreamValue() {
if (indexed()) {
return popCachedStream(precisionStep).setLongValue(value);
} else {
return null;
}
}
};
return new LongFieldMapper.CustomLongNumericField(this, value);
}
@Override public FieldDataType fieldDataType() {

31
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/DoubleFieldMapper.java
View File

@ -182,16 +182,7 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
}
}
final double fValue = value;
return new CustomNumericField(names.indexName(), indexed(), stored() ? Numbers.doubleToBytes(fValue) : null) {
@Override public TokenStream tokenStreamValue() {
if (indexed()) {
return popCachedStream(precisionStep).setDoubleValue(fValue);
} else {
return null;
}
}
};
return new CustomDoubleNumericField(this, value);
}
@Override public FieldDataType fieldDataType() {
@ -240,4 +231,24 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
builder.field("include_in_all", includeInAll);
}
}
public static class CustomDoubleNumericField extends CustomNumericField {
private final double number;
private final NumberFieldMapper mapper;
public CustomDoubleNumericField(NumberFieldMapper mapper, double number) {
super(mapper, mapper.stored() ? Numbers.doubleToBytes(number) : null);
this.mapper = mapper;
this.number = number;
}
@Override public TokenStream tokenStreamValue() {
if (isIndexed) {
return mapper.popCachedStream().setDoubleValue(number);
}
return null;
}
}
}

31
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/FloatFieldMapper.java
View File

@ -181,16 +181,7 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
}
}
final float fValue = value;
return new CustomNumericField(names.indexName(), indexed(), stored() ? Numbers.floatToBytes(fValue) : null) {
@Override public TokenStream tokenStreamValue() {
if (indexed()) {
return popCachedStream(precisionStep).setFloatValue(fValue);
} else {
return null;
}
}
};
return new CustomFloatNumericField(this, value);
}
@Override public FieldDataType fieldDataType() {
@ -240,4 +231,24 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
builder.field("include_in_all", includeInAll);
}
}
public static class CustomFloatNumericField extends CustomNumericField {
private final float number;
private final NumberFieldMapper mapper;
public CustomFloatNumericField(NumberFieldMapper mapper, float number) {
super(mapper, mapper.stored() ? Numbers.floatToBytes(number) : null);
this.mapper = mapper;
this.number = number;
}
@Override public TokenStream tokenStreamValue() {
if (isIndexed) {
return mapper.popCachedStream().setFloatValue(number);
}
return null;
}
}
}

31
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/IntegerFieldMapper.java
View File

@ -181,16 +181,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
}
}
final int fValue = value;
return new CustomNumericField(names.indexName(), indexed(), stored() ? Numbers.intToBytes(fValue) : null) {
@Override public TokenStream tokenStreamValue() {
if (indexed()) {
return popCachedStream(precisionStep).setIntValue(fValue);
} else {
return null;
}
}
};
return new CustomIntegerNumericField(this, value);
}
@Override public FieldDataType fieldDataType() {
@ -239,4 +230,24 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
builder.field("include_in_all", includeInAll);
}
}
public static class CustomIntegerNumericField extends CustomNumericField {
private final int number;
private final NumberFieldMapper mapper;
public CustomIntegerNumericField(NumberFieldMapper mapper, int number) {
super(mapper, mapper.stored() ? Numbers.intToBytes(number) : null);
this.mapper = mapper;
this.number = number;
}
@Override public TokenStream tokenStreamValue() {
if (isIndexed) {
return mapper.popCachedStream().setIntValue(number);
}
return null;
}
}
}

32
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/LongFieldMapper.java
View File

@ -180,17 +180,7 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
}
}
}
final long fValue = value;
return new CustomNumericField(names.indexName(), indexed(), stored() ? Numbers.longToBytes(value) : null) {
@Override public TokenStream tokenStreamValue() {
if (indexed()) {
return popCachedStream(precisionStep).setLongValue(fValue);
} else {
return null;
}
}
};
return new CustomLongNumericField(this, value);
}
@Override public FieldDataType fieldDataType() {
@ -239,4 +229,24 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
builder.field("include_in_all", includeInAll);
}
}
public static class CustomLongNumericField extends CustomNumericField {
private final long number;
private final NumberFieldMapper mapper;
public CustomLongNumericField(NumberFieldMapper mapper, long number) {
super(mapper, mapper.stored() ? Numbers.longToBytes(number) : null);
this.mapper = mapper;
this.number = number;
}
@Override public TokenStream tokenStreamValue() {
if (isIndexed) {
return mapper.popCachedStream().setLongValue(number);
}
return null;
}
}
}

114
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/NumberFieldMapper.java
View File

@ -20,25 +20,19 @@
package org.elasticsearch.index.mapper.xcontent;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.AbstractField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.StringHelper;
import org.elasticsearch.common.thread.ThreadLocals;
import org.elasticsearch.common.trove.TIntObjectHashMap;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.cache.field.data.FieldDataCache;
import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.MergeMappingException;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayDeque;
import java.util.Deque;
/**
* @author kimchy (shay.banon)
@ -85,16 +79,16 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
}
}
private static final ThreadLocal<ThreadLocals.CleanableValue<TIntObjectHashMap<Deque<CachedNumericTokenStream>>>> cachedStreams = new ThreadLocal<ThreadLocals.CleanableValue<TIntObjectHashMap<Deque<CachedNumericTokenStream>>>>() {
@Override protected ThreadLocals.CleanableValue<TIntObjectHashMap<Deque<CachedNumericTokenStream>>> initialValue() {
return new ThreadLocals.CleanableValue<TIntObjectHashMap<Deque<CachedNumericTokenStream>>>(new TIntObjectHashMap<Deque<CachedNumericTokenStream>>());
}
};
protected int precisionStep;
protected Boolean includeInAll;
private ThreadLocal<ThreadLocals.CleanableValue<NumericTokenStream>> tokenStream = new ThreadLocal<ThreadLocals.CleanableValue<NumericTokenStream>>() {
@Override protected ThreadLocals.CleanableValue<NumericTokenStream> initialValue() {
return new ThreadLocals.CleanableValue<NumericTokenStream>(new NumericTokenStream(precisionStep));
}
};
protected NumberFieldMapper(Names names, int precisionStep,
Field.Index index, Field.Store store,
float boost, boolean omitNorms, boolean omitTermFreqAndPositions,
@ -176,98 +170,22 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
@Override public abstract FieldDataType fieldDataType();
/**
* Removes a cached numeric token stream. The stream will be returned to the cached once it is used
* since it implements the end method.
*/
protected CachedNumericTokenStream popCachedStream(int precisionStep) {
Deque<CachedNumericTokenStream> deque = cachedStreams.get().get().get(precisionStep);
if (deque == null) {
deque = new ArrayDeque<CachedNumericTokenStream>();
cachedStreams.get().get().put(precisionStep, deque);
deque.add(new CachedNumericTokenStream(new NumericTokenStream(precisionStep), precisionStep));
}
if (deque.isEmpty()) {
deque.add(new CachedNumericTokenStream(new NumericTokenStream(precisionStep), precisionStep));
}
return deque.pollFirst();
}
/**
* A wrapper around a numeric stream allowing to reuse it by implementing the end method which returns
* this stream back to the thread local cache.
*/
protected static final class CachedNumericTokenStream extends TokenStream {
private final int precisionStep;
private final NumericTokenStream numericTokenStream;
public CachedNumericTokenStream(NumericTokenStream numericTokenStream, int precisionStep) {
super(numericTokenStream);
this.numericTokenStream = numericTokenStream;
this.precisionStep = precisionStep;
}
public void end() throws IOException {
numericTokenStream.end();
}
/**
* Close the input TokenStream.
*/
public void close() throws IOException {
numericTokenStream.close();
TIntObjectHashMap<Deque<CachedNumericTokenStream>> cached = cachedStreams.get().get();
if (cached != null) {
Deque<CachedNumericTokenStream> cachedDeque = cached.get(precisionStep);
if (cachedDeque != null) {
cachedDeque.add(this);
}
}
}
/**
* Reset the filter as well as the input TokenStream.
*/
public void reset() throws IOException {
numericTokenStream.reset();
}
@Override public boolean incrementToken() throws IOException {
return numericTokenStream.incrementToken();
}
public CachedNumericTokenStream setIntValue(int value) {
numericTokenStream.setIntValue(value);
return this;
}
public CachedNumericTokenStream setLongValue(long value) {
numericTokenStream.setLongValue(value);
return this;
}
public CachedNumericTokenStream setFloatValue(float value) {
numericTokenStream.setFloatValue(value);
return this;
}
public CachedNumericTokenStream setDoubleValue(double value) {
numericTokenStream.setDoubleValue(value);
return this;
}
protected NumericTokenStream popCachedStream() {
return tokenStream.get().get();
}
// used to we can use a numeric field in a document that is then parsed twice!
protected abstract static class CustomNumericField extends AbstractField {
public abstract static class CustomNumericField extends AbstractField {
public CustomNumericField(String name, boolean indexed, byte[] value) {
this.name = StringHelper.intern(name); // field names are interned
protected final NumberFieldMapper mapper;
public CustomNumericField(NumberFieldMapper mapper, byte[] value) {
this.mapper = mapper;
this.name = mapper.names().indexName();
fieldsData = value;
isIndexed = indexed;
isTokenized = indexed;
isIndexed = mapper.indexed();
isTokenized = mapper.indexed();
omitTermFreqAndPositions = true;
omitNorms = true;

31
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/ShortFieldMapper.java
View File

@ -180,16 +180,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
}
}
}
final short fValue = value;
return new CustomNumericField(names.indexName(), indexed(), stored() ? Numbers.shortToBytes(fValue) : null) {
@Override public TokenStream tokenStreamValue() {
if (indexed()) {
return popCachedStream(precisionStep).setIntValue(fValue);
} else {
return null;
}
}
};
return new CustomShortNumericField(this, value);
}
@Override public FieldDataType fieldDataType() {
@ -238,4 +229,24 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
builder.field("include_in_all", includeInAll);
}
}
public static class CustomShortNumericField extends CustomNumericField {
private final short number;
private final NumberFieldMapper mapper;
public CustomShortNumericField(NumberFieldMapper mapper, short number) {
super(mapper, mapper.stored() ? Numbers.shortToBytes(number) : null);
this.mapper = mapper;
this.number = number;
}
@Override public TokenStream tokenStreamValue() {
if (isIndexed) {
return mapper.popCachedStream().setIntValue(number);
}
return null;
}
}
}

16
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/ip/IpFieldMapper.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.index.mapper.xcontent.ip;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.Filter;
@ -40,10 +39,7 @@ import org.elasticsearch.index.cache.field.data.FieldDataCache;
import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.xcontent.MergeContext;
import org.elasticsearch.index.mapper.xcontent.NumberFieldMapper;
import org.elasticsearch.index.mapper.xcontent.ParseContext;
import org.elasticsearch.index.mapper.xcontent.XContentMapper;
import org.elasticsearch.index.mapper.xcontent.*;
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import java.io.IOException;
@ -219,15 +215,7 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
}
final long value = ipToLong(ipAsString);
return new CustomNumericField(names.indexName(), indexed(), stored() ? Numbers.longToBytes(value) : null) {
@Override public TokenStream tokenStreamValue() {
if (indexed()) {
return popCachedStream(precisionStep).setLongValue(value);
} else {
return null;
}
}
};
return new LongFieldMapper.CustomLongNumericField(this, value);
}
@Override public FieldDataType fieldDataType() {