honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-09 14:34:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow Date Fields to have a locale for date parsing

Browse Source 
Currently if somebody uses a date format that is locale dependend
date fields can only parse a single format depending on the nodes
host locale. This can cause lots of problems since nodes might have
different locales. ie. "E, d MMM yyyy HH:mm:ss Z" where you have
"Wed, 06 Dec 2000 02:55:00 -0800" for en_EN while
"Mi, 06 Dez 2000 02:55:00 -0800" for de_DE.

Closes #3047
This commit is contained in:
Simon Willnauer 2013-05-17 17:00:05 +02:00
parent e580507fbe
commit c4db582f26
6 changed files with 192 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/main/java/org/elasticsearch/common/joda/FormatDateTimeFormatter.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.common.joda;
import java.util.Locale;
import org.joda.time.format.DateTimeFormatter;
/**
@ -32,17 +34,20 @@ public class FormatDateTimeFormatter {
private final DateTimeFormatter parser;
private final DateTimeFormatter printer;
private final Locale locale;
public FormatDateTimeFormatter(String format, DateTimeFormatter parser) {
this(format, parser, parser);
public FormatDateTimeFormatter(String format, DateTimeFormatter parser, Locale locale) {
this(format, parser, parser, locale);
}
public FormatDateTimeFormatter(String format, DateTimeFormatter parser, DateTimeFormatter printer) {
public FormatDateTimeFormatter(String format, DateTimeFormatter parser, DateTimeFormatter printer, Locale locale) {
this.format = format;
this.parser = parser;
this.printer = printer;
this.locale = locale;
this.printer = locale == null ? printer : printer.withLocale(locale);
this.parser = locale == null ? parser : parser.withLocale(locale);
}
public String format() {
return format;
}
@ -54,4 +59,8 @@ public class FormatDateTimeFormatter {
public DateTimeFormatter printer() {
return this.printer;
}
public Locale locale() {
return locale;
}
}

13
src/main/java/org/elasticsearch/common/joda/Joda.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.common.joda;
import java.util.Locale;
import org.elasticsearch.common.Strings;
import org.joda.time.*;
import org.joda.time.field.DividedDateTimeField;
@ -31,10 +33,14 @@ import org.joda.time.format.*;
*/
public class Joda {
public static FormatDateTimeFormatter forPattern(String input) {
return forPattern(input, Locale.ROOT);
}
/**
* Parses a joda based pattern, including some named ones (similar to the built in Joda ISO ones).
*/
public static FormatDateTimeFormatter forPattern(String input) {
public static FormatDateTimeFormatter forPattern(String input, Locale locale) {
DateTimeFormatter formatter;
if ("basicDate".equals(input) || "basic_date".equals(input)) {
formatter = ISODateTimeFormat.basicDate();
@ -76,9 +82,10 @@ public class Joda {
formatter = ISODateTimeFormat.dateHourMinuteSecondMillis();
} else if ("dateOptionalTime".equals(input) || "date_optional_time".equals(input)) {
// in this case, we have a separate parser and printer since the dataOptionalTimeParser can't print
// this sucks we should use the root local by default and not be dependent on the node
return new FormatDateTimeFormatter(input,
ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC),
ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC));
ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC), locale);
} else if ("dateTime".equals(input) || "date_time".equals(input)) {
formatter = ISODateTimeFormat.dateTime();
} else if ("dateTimeNoMillis".equals(input) || "date_time_no_millis".equals(input)) {
@ -133,7 +140,7 @@ public class Joda {
formatter = builder.toFormatter();
}
}
return new FormatDateTimeFormatter(input, formatter.withZone(DateTimeZone.UTC));
return new FormatDateTimeFormatter(input, formatter.withZone(DateTimeZone.UTC), locale);
}

2
src/main/java/org/elasticsearch/index/mapper/Mapper.java
View File

@ -121,7 +121,7 @@ public interface Mapper extends ToXContent {
}
}
Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException;
Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException;
}
String name();

42
src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -51,6 +51,7 @@ import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@ -88,6 +89,8 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
protected FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
private Locale locale;
public Builder(String name) {
super(name, new FieldType(Defaults.FIELD_TYPE));
builder = this;
@ -115,17 +118,26 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
parseUpperInclusive = context.indexSettings().getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.PARSE_UPPER_INCLUSIVE);
}
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
if (locale != null && !locale.equals(dateTimeFormatter.locale())) {
// this sucks we should use the root local by default and not be dependent on the node if it is null?
dateTimeFormatter = new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale);
}
DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
precisionStep, boost, fieldType, nullValue,
timeUnit, parseUpperInclusive, ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
public Builder locale(Locale locale) {
this.locale = locale;
return this;
}
}
public static class TypeParser implements Mapper.TypeParser {
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
DateFieldMapper.Builder builder = dateField(name);
parseNumberField(builder, name, node, parserContext);
for (Map.Entry<String, Object> entry : node.entrySet()) {
@ -137,11 +149,34 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
builder.dateTimeFormatter(parseDateTimeFormatter(propName, propNode));
} else if (propName.equals("numeric_resolution")) {
builder.timeUnit(TimeUnit.valueOf(propNode.toString().toUpperCase()));
} else if (propName.equals("locale")) {
builder.locale(parseLocal(propNode.toString()));
}
}
return builder;
}
}
// public for test
public static Locale parseLocal(String locale) {
final String[] parts = locale.split("_", -1);
switch (parts.length) {
case 3:
// lang_country_variant
return new Locale(parts[0], parts[1], parts[2]);
case 2:
// lang_country
return new Locale(parts[0], parts[1]);
case 1:
if ("ROOT".equalsIgnoreCase(parts[0])) {
return Locale.ROOT;
}
// lang
return new Locale(parts[0]);
default:
throw new ElasticSearchIllegalArgumentException("Can't parse locale: [" + locale + "]");
}
}
protected final FormatDateTimeFormatter dateTimeFormatter;
@ -413,6 +448,9 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
if (timeUnit != Defaults.TIME_UNIT) {
builder.field("numeric_resolution", timeUnit.name().toLowerCase());
}
if (dateTimeFormatter.locale() != null) {
builder.field("locale", dateTimeFormatter.format());
}
}
private long parseStringValue(String value) {
@ -423,7 +461,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
long time = Long.parseLong(value);
return timeUnit.toMillis(time);
} catch (NumberFormatException e1) {
throw new MapperParsingException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e);
throw new MapperParsingException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number with locale [" + dateTimeFormatter.locale() +"]", e);
}
}
}

38
src/test/java/org/elasticsearch/test/integration/search/simple/SimpleSearchTests.java
View File

@ -30,6 +30,7 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
import static org.hamcrest.MatcherAssert.assertThat;
@ -160,4 +161,41 @@ public class SimpleSearchTests extends AbstractNodesTests {
searchResponse = client.prepareSearch("test").setQuery(QueryBuilders.queryString("field:[2010-01-03||+2d TO 2010-01-04||+2d]")).execute().actionGet();
assertThat(searchResponse.getHits().totalHits(), equalTo(2l));
}
@Test
public void localDependentDateTests() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test")
.addMapping("type1",
jsonBuilder().startObject()
.startObject("type1")
.startObject("properties")
.startObject("date_field")
.field("type", "date")
.field("format", "E, d MMM yyyy HH:mm:ss Z")
.field("locale", "de")
.endObject()
.endObject()
.endObject()
.endObject())
.setSettings(ImmutableSettings.settingsBuilder()).execute().actionGet();
for (int i = 0; i < 10; i++) {
client.prepareIndex("test", "type1", ""+i).setSource("date_field", "Mi, 06 Dez 2000 02:55:00 -0800").execute().actionGet();
client.prepareIndex("test", "type1", ""+(10+i)).setSource("date_field", "Do, 07 Dez 2000 02:55:00 -0800").execute().actionGet();
}
client.admin().indices().prepareRefresh().execute().actionGet();
for (int i = 0; i < 10; i++) {
SearchResponse searchResponse = client.prepareSearch("test")
.setQuery(QueryBuilders.rangeQuery("date_field").gte("Di, 05 Dez 2000 02:55:00 -0800").lte("Do, 07 Dez 2000 00:00:00 -0800"))
.execute().actionGet();
assertThat(searchResponse.getHits().totalHits(), equalTo(10l));
searchResponse = client.prepareSearch("test")
.setQuery(QueryBuilders.rangeQuery("date_field").gte( "Di, 05 Dez 2000 02:55:00 -0800").lte("Fr, 08 Dez 2000 00:00:00 -0800"))
.execute().actionGet();
assertThat(searchResponse.getHits().totalHits(), equalTo(20l));
}
}
}

94
src/test/java/org/elasticsearch/test/unit/index/mapper/date/SimpleDateMappingTests.java
View File

@ -19,8 +19,23 @@
package org.elasticsearch.test.unit.index.mapper.date;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.apache.lucene.analysis.NumericTokenStream.NumericTermAttribute;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.NumericRangeFilter;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentFactory;
@ -36,10 +51,6 @@ import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.testng.annotations.Test;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
@Test
public class SimpleDateMappingTests {
@ -51,7 +62,7 @@ public class SimpleDateMappingTests {
DocumentMapper defaultMapper = MapperTests.newParser().parse(mapping);
ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("date_field1", "2011/01/22")
.field("date_field2", "2011/01/22 00:00:00")
@ -61,7 +72,7 @@ public class SimpleDateMappingTests {
.endObject()
.bytes());
FieldMapper fieldMapper = defaultMapper.mappers().smartNameFieldMapper("date_field1");
FieldMapper<?> fieldMapper = defaultMapper.mappers().smartNameFieldMapper("date_field1");
assertThat(fieldMapper, instanceOf(DateFieldMapper.class));
fieldMapper = defaultMapper.mappers().smartNameFieldMapper("date_field2");
assertThat(fieldMapper, instanceOf(DateFieldMapper.class));
@ -73,6 +84,77 @@ public class SimpleDateMappingTests {
fieldMapper = defaultMapper.mappers().smartNameFieldMapper("wrong_date3");
assertThat(fieldMapper, instanceOf(StringFieldMapper.class));
}
@Test
public void testParseLocal() {
assertThat(Locale.GERMAN, equalTo(DateFieldMapper.parseLocal("de")));
assertThat(Locale.GERMANY, equalTo(DateFieldMapper.parseLocal("de_DE")));
assertThat(new Locale("de","DE","DE"), equalTo(DateFieldMapper.parseLocal("de_DE_DE")));
try {
DateFieldMapper.parseLocal("de_DE_DE_DE");
assert false;
} catch(ElasticSearchIllegalArgumentException ex) {
// expected
}
assertThat(Locale.ROOT, equalTo(DateFieldMapper.parseLocal("")));
assertThat(Locale.ROOT, equalTo(DateFieldMapper.parseLocal("ROOT")));
}
@Test
public void testLocale() throws IOException {
String mapping = XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("date_field_default")
.field("type", "date")
.field("format", "E, d MMM yyyy HH:mm:ss Z")
.endObject()
.startObject("date_field_en")
.field("type", "date")
.field("format", "E, d MMM yyyy HH:mm:ss Z")
.field("locale", "EN")
.endObject()
.startObject("date_field_de")
.field("type", "date")
.field("format", "E, d MMM yyyy HH:mm:ss Z")
.field("locale", "DE_de")
.endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = MapperTests.newParser().parse(mapping);
ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("date_field_en", "Wed, 06 Dec 2000 02:55:00 -0800")
.field("date_field_de", "Mi, 06 Dez 2000 02:55:00 -0800")
.field("date_field_default", "Wed, 06 Dec 2000 02:55:00 -0800") // check default - root?
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("date_field_en").tokenStream(defaultMapper.indexAnalyzer()), notNullValue());
assertThat(doc.rootDoc().getField("date_field_de").tokenStream(defaultMapper.indexAnalyzer()), notNullValue());
TokenStream tokenStream = doc.rootDoc().getField("date_field_en").tokenStream(defaultMapper.indexAnalyzer());
tokenStream.reset();
NumericTermAttribute nta = tokenStream.addAttribute(NumericTermAttribute.class);
List<Long> values = new ArrayList<Long>();
while(tokenStream.incrementToken()) {
values.add(nta.getRawValue());
}
tokenStream = doc.rootDoc().getField("date_field_de").tokenStream(defaultMapper.indexAnalyzer());
tokenStream.reset();
nta = tokenStream.addAttribute(NumericTermAttribute.class);
int pos = 0;
while(tokenStream.incrementToken()) {
assertThat(values.get(pos++), equalTo(nta.getRawValue()));
}
assertThat(pos, equalTo(values.size()));
}
@Test
public void testTimestampAsDate() throws Exception {