honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-22 12:56:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add "locale" parameter to query_string and simple_query_string

Browse Source 
Fixes #5128

Remove java 7 specific Locale functions, add "coming[1.1.0]" to documentation

add LocaleUtils utility class for dealing with Locale functions
This commit is contained in:
Lee Hinman 2014-02-14 12:34:16 -07:00
parent 96d028e721
commit 8f8cc7205d
13 changed files with 151 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/reference/query-dsl/queries/query-string-query.asciidoc
View File

@ -71,6 +71,9 @@ both>>.
|`lenient` |If set to `true` will cause format based failures (like
providing text to a numeric field) to be ignored.
|`locale` | coming[1.1.0] Locale that should be used for string conversions.
Defaults to `ROOT`.
|=======================================================================
When a multi term query is being generated, one can control how it gets

3
docs/reference/query-dsl/queries/simple-query-string-query.asciidoc
View File

@ -43,6 +43,9 @@ enable. Defaults to `ALL`.
|`lowercase_expanded_terms` | Whether terms of prefix and fuzzy queries are to
be automatically lower-cased or not (since they are not analyzed). Defaults to
true.
|`locale` | coming[1.1.0] Locale that should be used for string conversions.
Defaults to `ROOT`.
|=======================================================================
[float]

1
src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
View File

@ -126,6 +126,7 @@ public class MapperQueryParser extends QueryParser {
setDefaultOperator(settings.defaultOperator());
setFuzzyMinSim(settings.fuzzyMinSim());
setFuzzyPrefixLength(settings.fuzzyPrefixLength());
setLocale(settings.locale());
this.analyzeWildcard = settings.analyzeWildcard();
}

14
src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.search.MultiTermQuery;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
/**
*
@ -59,6 +60,7 @@ public class QueryParserSettings {
private MultiTermQuery.RewriteMethod rewriteMethod = MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT;
private String minimumShouldMatch;
private boolean lenient;
private Locale locale;
List<String> fields = null;
@ -296,6 +298,14 @@ public class QueryParserSettings {
this.useDisMax = useDisMax;
}
public void locale(Locale locale) {
this.locale = locale;
}
public Locale locale() {
return this.locale;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
@ -336,6 +346,9 @@ public class QueryParserSettings {
if (lenient != that.lenient) {
return false;
}
if (locale != null ? !locale.equals(that.locale) : that.locale != null) {
return false;
}
if (Float.compare(that.tieBreaker, tieBreaker) != 0) return false;
if (useDisMax != that.useDisMax) return false;
@ -371,6 +384,7 @@ public class QueryParserSettings {
result = 31 * result + (boosts != null ? boosts.hashCode() : 0);
result = 31 * result + (tieBreaker != +0.0f ? Float.floatToIntBits(tieBreaker) : 0);
result = 31 * result + (useDisMax ? 1 : 0);
result = 31 * result + (locale != null ? locale.hashCode() : 0);
return result;
}
}

61
src/main/java/org/elasticsearch/common/util/LocaleUtils.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.util;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import java.util.Locale;
/**
* Utilities for for dealing with {@link Locale} objects
*/
public class LocaleUtils {
/**
* Parse the string describing a locale into a {@link Locale} object
*/
public static Locale parse(String localeStr) {
final String[] parts = localeStr.split("_", -1);
switch (parts.length) {
case 3:
// lang_country_variant
return new Locale(parts[0], parts[1], parts[2]);
case 2:
// lang_country
return new Locale(parts[0], parts[1]);
case 1:
if ("ROOT".equalsIgnoreCase(parts[0])) {
return Locale.ROOT;
}
// lang
return new Locale(parts[0]);
default:
throw new ElasticsearchIllegalArgumentException("Can't parse locale: [" + localeStr + "]");
}
}
/**
* Return a string for a {@link Locale} object
*/
public static String toString(Locale locale) {
// JAVA7 - use .toLanguageTag instead of .toString()
return locale.toString();
}
}

24
src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -38,6 +38,7 @@ import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.lucene.search.NoCacheFilter;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.util.LocaleUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -157,34 +158,13 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
} else if (propName.equals("numeric_resolution")) {
builder.timeUnit(TimeUnit.valueOf(propNode.toString().toUpperCase(Locale.ROOT)));
} else if (propName.equals("locale")) {
builder.locale(parseLocale(propNode.toString()));
builder.locale(LocaleUtils.parse(propNode.toString()));
}
}
return builder;
}
}
// public for test
public static Locale parseLocale(String locale) {
final String[] parts = locale.split("_", -1);
switch (parts.length) {
case 3:
// lang_country_variant
return new Locale(parts[0], parts[1], parts[2]);
case 2:
// lang_country
return new Locale(parts[0], parts[1]);
case 1:
if ("ROOT".equalsIgnoreCase(parts[0])) {
return Locale.ROOT;
}
// lang
return new Locale(parts[0]);
default:
throw new ElasticsearchIllegalArgumentException("Can't parse locale: [" + locale + "]");
}
}
protected FormatDateTimeFormatter dateTimeFormatter;
// Triggers rounding up of the upper bound for range queries and filters if

10
src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
View File

@ -65,6 +65,8 @@ public class QueryStringQueryBuilder extends BaseQueryBuilder implements Boostab
private Boolean analyzeWildcard;
private Locale locale;
private float boost = -1;
@ -312,6 +314,11 @@ public class QueryStringQueryBuilder extends BaseQueryBuilder implements Boostab
return this;
}
public QueryStringQueryBuilder locale(Locale locale) {
this.locale = locale;
return this;
}
@Override
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(QueryStringQueryParser.NAME);
@ -392,6 +399,9 @@ public class QueryStringQueryBuilder extends BaseQueryBuilder implements Boostab
if (queryName != null) {
builder.field("_name", queryName);
}
if (locale != null) {
builder.field("locale", locale.toString());
}
builder.endObject();
}
}

6
src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java
View File

@ -32,11 +32,13 @@ import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.util.LocaleUtils;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.query.support.QueryParsers;
import java.io.IOException;
import java.util.Locale;
import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfNeeded;
import static org.elasticsearch.common.lucene.search.Queries.optimizeQuery;
@ -73,6 +75,7 @@ public class QueryStringQueryParser implements QueryParser {
qpSettings.lenient(parseContext.queryStringLenient());
qpSettings.analyzeWildcard(defaultAnalyzeWildcard);
qpSettings.allowLeadingWildcard(defaultAllowLeadingWildcard);
qpSettings.locale(Locale.ROOT);
String currentFieldName = null;
XContentParser.Token token;
@ -186,6 +189,9 @@ public class QueryStringQueryParser implements QueryParser {
qpSettings.quoteFieldSuffix(parser.textOrNull());
} else if ("lenient".equalsIgnoreCase(currentFieldName)) {
qpSettings.lenient(parser.booleanValue());
} else if ("locale".equals(currentFieldName)) {
String localeStr = parser.text();
qpSettings.locale(LocaleUtils.parse(localeStr));
} else if ("_name".equals(currentFieldName)) {
queryName = parser.text();
} else {

8
src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
View File

@ -39,11 +39,13 @@ import static org.elasticsearch.index.query.support.QueryParsers.wrapSmartNameQu
public class SimpleQueryParser extends XSimpleQueryParser {
private final boolean lowercaseExpandedTerms;
private final Locale locale;
/** Creates a new parser with custom flags used to enable/disable certain features. */
public SimpleQueryParser(Analyzer analyzer, Map<String, Float> weights, int flags, boolean lowercaseExpandedTerms) {
public SimpleQueryParser(Analyzer analyzer, Map<String, Float> weights, int flags, Locale locale, boolean lowercaseExpandedTerms) {
super(analyzer, weights, flags);
this.lowercaseExpandedTerms = lowercaseExpandedTerms;
this.locale = locale;
}
/**
@ -53,7 +55,7 @@ public class SimpleQueryParser extends XSimpleQueryParser {
@Override
public Query newFuzzyQuery(String text, int fuzziness) {
if (lowercaseExpandedTerms) {
text = text.toLowerCase(Locale.ROOT);
text = text.toLowerCase(locale);
}
return super.newFuzzyQuery(text, fuzziness);
}
@ -65,7 +67,7 @@ public class SimpleQueryParser extends XSimpleQueryParser {
@Override
public Query newPrefixQuery(String text) {
if (lowercaseExpandedTerms) {
text = text.toLowerCase(Locale.ROOT);
text = text.toLowerCase(locale);
}
return super.newPrefixQuery(text);
}

10
src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
View File

@ -37,6 +37,7 @@ public class SimpleQueryStringBuilder extends BaseQueryBuilder {
private final String queryText;
private int flags = -1;
private Boolean lowercaseExpandedTerms;
private Locale locale;
/**
* Operators for the default_operator
@ -107,6 +108,11 @@ public class SimpleQueryStringBuilder extends BaseQueryBuilder {
return this;
}
public SimpleQueryStringBuilder locale(Locale locale) {
this.locale = locale;
return this;
}
@Override
public void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(SimpleQueryStringParser.NAME);
@ -143,6 +149,10 @@ public class SimpleQueryStringBuilder extends BaseQueryBuilder {
builder.field("lowercase_expanded_terms", lowercaseExpandedTerms);
}
if (locale != null) {
builder.field("locale", locale.toString());
}
builder.endObject();
}
}

8
src/main/java/org/elasticsearch/index/query/SimpleQueryStringParser.java
View File

@ -27,12 +27,14 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.LocaleUtils;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.MapperService;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
/**
@ -91,6 +93,7 @@ public class SimpleQueryStringParser implements QueryParser {
BooleanClause.Occur defaultOperator = null;
Analyzer analyzer = null;
boolean lowercaseExpandedTerms = true;
Locale locale = Locale.ROOT;
int flags = -1;
XContentParser.Token token;
@ -168,6 +171,9 @@ public class SimpleQueryStringParser implements QueryParser {
flags = SimpleQueryStringFlag.ALL.value();
}
}
} else if ("locale".equals(currentFieldName)) {
String localeStr = parser.text();
locale = LocaleUtils.parse(localeStr);
} else if ("lowercase_expanded_terms".equals(currentFieldName)) {
lowercaseExpandedTerms = parser.booleanValue();
} else {
@ -199,7 +205,7 @@ public class SimpleQueryStringParser implements QueryParser {
if (fieldsAndWeights == null) {
fieldsAndWeights = Collections.singletonMap(field, 1.0F);
}
SimpleQueryParser sqp = new SimpleQueryParser(analyzer, fieldsAndWeights, flags, lowercaseExpandedTerms);
SimpleQueryParser sqp = new SimpleQueryParser(analyzer, fieldsAndWeights, flags, locale, lowercaseExpandedTerms);
if (defaultOperator != null) {
sqp.setDefaultOperator(defaultOperator);

13
src/test/java/org/elasticsearch/index/mapper/date/SimpleDateMappingTests.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.search.NumericRangeFilter;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.LocaleUtils;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
@ -81,18 +82,18 @@ public class SimpleDateMappingTests extends ElasticsearchTestCase {
@Test
public void testParseLocal() {
assertThat(Locale.GERMAN, equalTo(DateFieldMapper.parseLocale("de")));
assertThat(Locale.GERMANY, equalTo(DateFieldMapper.parseLocale("de_DE")));
assertThat(new Locale("de","DE","DE"), equalTo(DateFieldMapper.parseLocale("de_DE_DE")));
assertThat(Locale.GERMAN, equalTo(LocaleUtils.parse("de")));
assertThat(Locale.GERMANY, equalTo(LocaleUtils.parse("de_DE")));
assertThat(new Locale("de","DE","DE"), equalTo(LocaleUtils.parse("de_DE_DE")));
try {
DateFieldMapper.parseLocale("de_DE_DE_DE");
LocaleUtils.parse("de_DE_DE_DE");
fail();
} catch(ElasticsearchIllegalArgumentException ex) {
// expected
}
assertThat(Locale.ROOT, equalTo(DateFieldMapper.parseLocale("")));
assertThat(Locale.ROOT, equalTo(DateFieldMapper.parseLocale("ROOT")));
assertThat(Locale.ROOT, equalTo(LocaleUtils.parse("")));
assertThat(Locale.ROOT, equalTo(LocaleUtils.parse("ROOT")));
}
@Test

22
src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java
View File

@ -46,6 +46,7 @@ import org.joda.time.format.ISODateTimeFormat;
import org.junit.Test;
import java.io.IOException;
import java.util.Locale;
import java.util.Random;
import java.util.concurrent.ExecutionException;
@ -2003,6 +2004,27 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
assertHitCount(searchResponse, 0l);
}
@Test
public void testQueryStringLocale() {
assertAcked(client().admin().indices().prepareCreate("test").setSettings(SETTING_NUMBER_OF_SHARDS, 1));
client().prepareIndex("test", "type1", "1").setSource("body", "bılly").get();
refresh();
SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryString("BILL*")).get();
assertHitCount(searchResponse, 0l);
searchResponse = client().prepareSearch().setQuery(queryString("body:BILL*")).get();
assertHitCount(searchResponse, 0l);
searchResponse = client().prepareSearch().setQuery(
simpleQueryString("BILL*").locale(new Locale("tr", "TR"))).get();
assertHitCount(searchResponse, 1l);
assertSearchHits(searchResponse, "1");
searchResponse = client().prepareSearch().setQuery(
queryString("body:BILL*").locale(new Locale("tr", "TR"))).get();
assertHitCount(searchResponse, 1l);
assertSearchHits(searchResponse, "1");
}
@Test
public void testNestedFieldSimpleQueryString() throws IOException {
assertAcked(client().admin().indices().prepareCreate("test").setSettings(SETTING_NUMBER_OF_SHARDS, 1)