honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Date Range: Inclusive upper range does not round up properly, closes #1463.

This commit is contained in:
Shay Banon 2011-11-14 17:16:42 +02:00
parent ed281fbfd5
commit 73ba30b18a
8 changed files with 117 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View File

@ -140,14 +140,18 @@ public class DocumentMapper implements ToXContent {
private final String index;
@Nullable private final Settings indexSettings;
private final RootObjectMapper rootObjectMapper;
private ImmutableMap<String, Object> meta = ImmutableMap.of();
private Mapper.BuilderContext builderContext = new Mapper.BuilderContext(new ContentPath(1));
private final Mapper.BuilderContext builderContext;
public Builder(String index, @Nullable Settings indexSettings, RootObjectMapper.Builder builder) {
this.index = index;
this.indexSettings = indexSettings;
this.builderContext = new Mapper.BuilderContext(indexSettings, new ContentPath(1));
this.rootObjectMapper = builder.build(builderContext);
IdFieldMapper idFieldMapper = new IdFieldMapper();
if (indexSettings != null) {
@ -203,7 +207,7 @@ public class DocumentMapper implements ToXContent {
public DocumentMapper build(DocumentMapperParser docMapperParser) {
Preconditions.checkNotNull(rootObjectMapper, "Mapper builder must have the root object mapper set");
return new DocumentMapper(index, docMapperParser, rootObjectMapper, meta,
return new DocumentMapper(index, indexSettings, docMapperParser, rootObjectMapper, meta,
indexAnalyzer, searchAnalyzer,
rootMappers);
}
@ -212,12 +216,14 @@ public class DocumentMapper implements ToXContent {
private ThreadLocal<ParseContext> cache = new ThreadLocal<ParseContext>() {
@Override protected ParseContext initialValue() {
return new ParseContext(index, docMapperParser, DocumentMapper.this, new ContentPath(0));
return new ParseContext(index, indexSettings, docMapperParser, DocumentMapper.this, new ContentPath(0));
}
};
private final String index;
private final Settings indexSettings;
private final String type;
private final DocumentMapperParser docMapperParser;
@ -250,12 +256,13 @@ public class DocumentMapper implements ToXContent {
private final Object mutex = new Object();
public DocumentMapper(String index, DocumentMapperParser docMapperParser,
public DocumentMapper(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser,
RootObjectMapper rootObjectMapper,
ImmutableMap<String, Object> meta,
NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
Map<Class<? extends RootMapper>, RootMapper> rootMappers) {
this.index = index;
this.indexSettings = indexSettings;
this.type = rootObjectMapper.name();
this.docMapperParser = docMapperParser;
this.meta = meta;

10
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/Mapper.java
View File

@ -19,8 +19,10 @@
package org.elasticsearch.index.mapper;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.NotThreadSafe;
import org.elasticsearch.common.util.concurrent.ThreadSafe;
import org.elasticsearch.common.xcontent.ToXContent;
@ -39,15 +41,21 @@ public interface Mapper extends ToXContent {
@NotThreadSafe
public static class BuilderContext {
private final Settings indexSettings;
private final ContentPath contentPath;
public BuilderContext(ContentPath contentPath) {
public BuilderContext(@Nullable Settings indexSettings, ContentPath contentPath) {
this.contentPath = contentPath;
this.indexSettings = indexSettings;
}
public ContentPath path() {
return this.contentPath;
}
@Nullable public Settings indexSettings() {
return this.indexSettings;
}
}
@NotThreadSafe

13
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
View File

@ -22,7 +22,9 @@ package org.elasticsearch.index.mapper;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.lucene.all.AllEntries;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.NotThreadSafe;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.AnalysisService;
@ -53,7 +55,9 @@ public class ParseContext {
private Analyzer analyzer;
private String index;
private final String index;
@Nullable private final Settings indexSettings;
private SourceToParse sourceToParse;
private byte[] source;
@ -78,8 +82,9 @@ public class ParseContext {
private AllEntries allEntries = new AllEntries();
public ParseContext(String index, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) {
public ParseContext(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) {
this.index = index;
this.indexSettings = indexSettings;
this.docMapper = docMapper;
this.docMapperParser = docMapperParser;
this.path = path;
@ -128,6 +133,10 @@ public class ParseContext {
return this.index;
}
@Nullable public Settings indexSettings() {
return this.indexSettings;
}
public String type() {
return sourceToParse.type();
}

38
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -30,6 +30,8 @@ import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.joda.time.DateTimeZone;
import org.elasticsearch.common.joda.time.MutableDateTime;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
@ -64,6 +66,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
public static final String NULL_VALUE = null;
public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
public static final boolean PARSE_UPPER_INCLUSIVE = true;
}
public static class Builder extends NumberFieldMapper.Builder<Builder, DateFieldMapper> {
@ -95,8 +98,12 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
}
@Override public DateFieldMapper build(BuilderContext context) {
boolean parseUpperInclusive = Defaults.PARSE_UPPER_INCLUSIVE;
if (context.indexSettings() != null) {
parseUpperInclusive = context.indexSettings().getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.PARSE_UPPER_INCLUSIVE);
}
DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
precisionStep, fuzzyFactor, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue, timeUnit);
precisionStep, fuzzyFactor, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue, timeUnit, parseUpperInclusive);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -123,6 +130,8 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
protected final FormatDateTimeFormatter dateTimeFormatter;
private final boolean parseUpperInclusive;
private String nullValue;
protected final TimeUnit timeUnit;
@ -130,13 +139,14 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, String fuzzyFactor,
Field.Index index, Field.Store store,
float boost, boolean omitNorms, boolean omitTermFreqAndPositions,
String nullValue, TimeUnit timeUnit) {
String nullValue, TimeUnit timeUnit, boolean parseUpperInclusive) {
super(names, precisionStep, fuzzyFactor, index, store, boost, omitNorms, omitTermFreqAndPositions,
new NamedAnalyzer("_date/" + precisionStep, new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())),
new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())));
this.dateTimeFormatter = dateTimeFormatter;
this.nullValue = nullValue;
this.timeUnit = timeUnit;
this.parseUpperInclusive = parseUpperInclusive;
}
@Override protected double parseFuzzyFactor(String fuzzyFactor) {
@ -212,21 +222,21 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
@Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) {
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
lowerTerm == null ? null : parseStringValue(lowerTerm),
upperTerm == null ? null : parseStringValue(upperTerm),
upperTerm == null ? null : includeUpper ? parseUpperInclusiveStringValue(upperTerm) : parseStringValue(upperTerm),
includeLower, includeUpper);
}
@Override public Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) {
return NumericRangeFilter.newLongRange(names.indexName(), precisionStep,
lowerTerm == null ? null : parseStringValue(lowerTerm),
upperTerm == null ? null : parseStringValue(upperTerm),
upperTerm == null ? null : includeUpper ? parseUpperInclusiveStringValue(upperTerm) : parseStringValue(upperTerm),
includeLower, includeUpper);
}
@Override public Filter rangeFilter(FieldDataCache fieldDataCache, String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) {
return NumericRangeFieldDataFilter.newLongRange(fieldDataCache, names.indexName(),
lowerTerm == null ? null : parseStringValue(lowerTerm),
upperTerm == null ? null : parseStringValue(upperTerm),
upperTerm == null ? null : includeUpper ? parseUpperInclusiveStringValue(upperTerm) : parseStringValue(upperTerm),
includeLower, includeUpper);
}
@ -363,4 +373,22 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
}
}
}
protected long parseUpperInclusiveStringValue(String value) {
if (!parseUpperInclusive) {
return parseStringValue(value);
}
try {
MutableDateTime dateTime = new MutableDateTime(3000, 12, 31, 23, 59, 59, 999, DateTimeZone.UTC);
dateTimeFormatter.parser().parseInto(dateTime, value, 0);
return dateTime.getMillis();
} catch (RuntimeException e) {
try {
long time = Long.parseLong(value);
return timeUnit.toMillis(time);
} catch (NumberFormatException e1) {
throw new MapperParsingException("failed to parse date field, tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e);
}
}
}
}

12
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View File

@ -90,7 +90,11 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
}
@Override public TimestampFieldMapper build(BuilderContext context) {
return new TimestampFieldMapper(store, index, enabled, path, dateTimeFormatter);
boolean parseUpperInclusive = Defaults.PARSE_UPPER_INCLUSIVE;
if (context.indexSettings() != null) {
parseUpperInclusive = context.indexSettings().getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.PARSE_UPPER_INCLUSIVE);
}
return new TimestampFieldMapper(store, index, enabled, path, dateTimeFormatter, parseUpperInclusive);
}
}
@ -119,13 +123,13 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
private final String path;
public TimestampFieldMapper() {
this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER);
this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.PARSE_UPPER_INCLUSIVE);
}
protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path, FormatDateTimeFormatter dateTimeFormatter) {
protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path, FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS,
Defaults.OMIT_TERM_FREQ_AND_POSITIONS, Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/);
Defaults.OMIT_TERM_FREQ_AND_POSITIONS, Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/, parseUpperInclusive);
this.enabled = enabled;
this.path = path;
}

4
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java
View File

@ -510,7 +510,7 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
}
// remove the current field name from path, since the object builder adds it as well...
context.path().remove();
BuilderContext builderContext = new BuilderContext(context.path());
BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path());
objectMapper = builder.build(builderContext);
putMapper(objectMapper);
// now re add it
@ -595,7 +595,7 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
mapper = mappers.get(currentFieldName);
if (mapper == null) {
newMapper = true;
BuilderContext builderContext = new BuilderContext(context.path());
BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path());
if (token == XContentParser.Token.VALUE_STRING) {
boolean resolved = false;

10
modules/elasticsearch/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java
View File

@ -61,6 +61,16 @@ public class SimpleJodaTests {
assertThat(millis, equalTo(0l));
}
@Test public void testUpperBound() {
MutableDateTime dateTime = new MutableDateTime(3000, 12, 31, 23, 59, 59, 999, DateTimeZone.UTC);
DateTimeFormatter formatter = ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC);
String value = "2000-01-01";
int i = formatter.parseInto(dateTime, value, 0);
assertThat(i, equalTo(value.length()));
assertThat(dateTime.toString(), equalTo("2000-01-01T23:59:59.999Z"));
}
@Test public void testIsoDateFormatDateOptionalTimeUTC() {
DateTimeFormatter formatter = ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC);
long millis = formatter.parseMillis("1970-01-01T00:00:00Z");

44
modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/ip/IpSearchTests.java → modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/simple/SimpleSearchTests.java
View File

@ -17,12 +17,13 @@
* under the License.
*/
package org.elasticsearch.test.integration.search.ip;
package org.elasticsearch.test.integration.search.simple;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.test.integration.AbstractNodesTests;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@ -32,10 +33,7 @@ import static org.elasticsearch.index.query.QueryBuilders.*;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/**
* @author kimchy (shay.banon)
*/
public class IpSearchTests extends AbstractNodesTests {
public class SimpleSearchTests extends AbstractNodesTests {
private Client client;
@ -53,12 +51,8 @@ public class IpSearchTests extends AbstractNodesTests {
return client("node1");
}
@Test public void filterExistsMissingTests() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// ignore
}
@Test public void simpleIpTests() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 1)).execute().actionGet();
@ -77,4 +71,32 @@ public class IpSearchTests extends AbstractNodesTests {
assertThat(search.hits().totalHits(), equalTo(1l));
}
@Test public void simpleDateRangeWithUpperInclusiveEnabledTests() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder()).execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("field", "2010-01-05T02:00").execute().actionGet();
client.prepareIndex("test", "type1", "2").setSource("field", "2010-01-06T02:00").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
// test include upper on ranges to include the full day on the upper bound
SearchResponse searchResponse = client.prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05").lte("2010-01-06")).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(2l));
searchResponse = client.prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05").lt("2010-01-06")).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(1l));
}
@Test public void simpleDateRangeWithUpperInclusiveDisabledTests() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.mapping.date.parse_upper_inclusive", false)).execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("field", "2010-01-05T02:00").execute().actionGet();
client.prepareIndex("test", "type1", "2").setSource("field", "2010-01-06T02:00").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
// test include upper on ranges to include the full day on the upper bound (disabled here though...)
SearchResponse searchResponse = client.prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05").lte("2010-01-06")).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(1l));
searchResponse = client.prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05").lt("2010-01-06")).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(1l));
}
}