honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

more parsers

This commit is contained in:
Simon Willnauer 2015-11-03 15:51:39 +01:00 committed by Areek Zillur
parent f5560a3087
commit 6f9a486071
6 changed files with 142 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
View File

@ -214,8 +214,16 @@ public final class ObjectParser<Value, Context> implements BiFunction<XContentPa
private final <T> List<T> parseArray(XContentParser parser, IOSupplier<T> supplier) throws IOException {
List<T> list = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
list.add(supplier.get());
if (parser.currentToken().isValue()) {
list.add(supplier.get()); // single value
} else {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
if (parser.currentToken().isValue()) {
list.add(supplier.get());
} else {
throw new IllegalStateException("expected value but got [" + parser.currentToken() + "]");
}
}
}
return list;
}
@ -224,6 +232,19 @@ public final class ObjectParser<Value, Context> implements BiFunction<XContentPa
declareField((p, v, c) -> consumer.accept(v, objectParser.apply(p, c)), field, ValueType.OBJECT);
}
public <T> void declareObjectOrDefault(BiConsumer<Value, T> consumer, BiFunction<XContentParser, Context, T> objectParser, Supplier<T> defaultValue, ParseField field) {
declareField((p, v, c) -> {
if (p.currentToken() == XContentParser.Token.VALUE_BOOLEAN) {
if (p.booleanValue()) {
consumer.accept(v, defaultValue.get());
}
} else {
consumer.accept(v, objectParser.apply(p, c));
}
}, field, ValueType.OBJECT_OR_BOOLEAN);
}
public void declareFloat(BiConsumer<Value, Float> consumer, ParseField field) {
declareField((p, v, c) -> consumer.accept(v, p.floatValue()), field, ValueType.FLOAT);
}
@ -300,13 +321,14 @@ public final class ObjectParser<Value, Context> implements BiFunction<XContentPa
DOUBLE(EnumSet.of(XContentParser.Token.VALUE_NUMBER, XContentParser.Token.VALUE_STRING)),
LONG(EnumSet.of(XContentParser.Token.VALUE_NUMBER, XContentParser.Token.VALUE_STRING)),
INT(EnumSet.of(XContentParser.Token.VALUE_NUMBER, XContentParser.Token.VALUE_STRING)),
BOOLEAN(EnumSet.of(XContentParser.Token.VALUE_BOOLEAN)), STRING_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY)),
FLOAT_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY)),
DOUBLE_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY)),
LONG_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY)),
INT_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY)),
BOOLEAN_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY)),
BOOLEAN(EnumSet.of(XContentParser.Token.VALUE_BOOLEAN)), STRING_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY, XContentParser.Token.VALUE_STRING)),
FLOAT_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY, XContentParser.Token.VALUE_NUMBER, XContentParser.Token.VALUE_STRING)),
DOUBLE_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY, XContentParser.Token.VALUE_NUMBER, XContentParser.Token.VALUE_STRING)),
LONG_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY, XContentParser.Token.VALUE_NUMBER, XContentParser.Token.VALUE_STRING)),
INT_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY, XContentParser.Token.VALUE_NUMBER, XContentParser.Token.VALUE_STRING)),
BOOLEAN_ARRAY(EnumSet.of(XContentParser.Token.START_ARRAY, XContentParser.Token.VALUE_BOOLEAN)),
OBJECT(EnumSet.of(XContentParser.Token.START_OBJECT)),
OBJECT_OR_BOOLEAN(EnumSet.of(XContentParser.Token.START_OBJECT, XContentParser.Token.VALUE_BOOLEAN)),
VALUE(EnumSet.of(XContentParser.Token.VALUE_BOOLEAN, XContentParser.Token.VALUE_NULL ,XContentParser.Token.VALUE_EMBEDDED_OBJECT,XContentParser.Token.VALUE_NUMBER,XContentParser.Token.VALUE_STRING));
private final EnumSet<XContentParser.Token> tokens;

136
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java
View File

@ -18,9 +18,9 @@
*/
package org.elasticsearch.search.suggest.completion;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.HasContextAndHeaders;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.Fuzziness;
@ -28,6 +28,7 @@ import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
@ -40,8 +41,6 @@ import org.elasticsearch.search.suggest.completion.context.ContextMappings;
import java.io.IOException;
import java.util.*;
import static org.elasticsearch.search.suggest.SuggestUtils.parseSuggestContext;
/**
* Parses query options for {@link CompletionSuggester}
*
@ -72,6 +71,59 @@ import static org.elasticsearch.search.suggest.SuggestUtils.parseSuggestContext;
*/
public class CompletionSuggestParser implements SuggestContextParser {
private static ObjectParser<CompletionSuggestionContext, ContextAndSuggest> TLP_PARSER = new ObjectParser<>("completion", null);
private static ObjectParser<CompletionSuggestionBuilder.RegexOptionsBuilder, ContextAndSuggest> REGEXP_PARSER = new ObjectParser<>("regexp", CompletionSuggestionBuilder.RegexOptionsBuilder::new);
private static ObjectParser<CompletionSuggestionBuilder.FuzzyOptionsBuilder, ContextAndSuggest> FUZZY_PARSER = new ObjectParser<>("fuzzy", CompletionSuggestionBuilder.FuzzyOptionsBuilder::new);
static {
FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setFuzzyMinLength, new ParseField("min_length"));
FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setMaxDeterminizedStates, new ParseField("max_determinized_states"));
FUZZY_PARSER.declareBoolean(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setUnicodeAware, new ParseField("unicode_aware"));
FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setFuzzyPrefixLength, new ParseField("prefix_length"));
FUZZY_PARSER.declareBoolean(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setTranspositions, new ParseField("transpositions"));
FUZZY_PARSER.declareValue((a, b) -> {
try {
a.setFuzziness(Fuzziness.parse(b).asDistance());
} catch (IOException e) {
throw new ElasticsearchException(e);
}
}, new ParseField("fuzziness"));
REGEXP_PARSER.declareInt(CompletionSuggestionBuilder.RegexOptionsBuilder::setMaxDeterminizedStates, new ParseField("max_determinized_states"));
REGEXP_PARSER.declareStringOrNull(CompletionSuggestionBuilder.RegexOptionsBuilder::setFlags, new ParseField("flags"));
TLP_PARSER.declareStringArray(CompletionSuggestionContext::setPayloadFields, new ParseField("payload"));
TLP_PARSER.declareObjectOrDefault(CompletionSuggestionContext::setFuzzyOptionsBuilder, FUZZY_PARSER, CompletionSuggestionBuilder.FuzzyOptionsBuilder::new, new ParseField("fuzzy"));
TLP_PARSER.declareObject(CompletionSuggestionContext::setRegexOptionsBuilder, REGEXP_PARSER, new ParseField("regexp"));
TLP_PARSER.declareString(SuggestionSearchContext.SuggestionContext::setField, new ParseField("field"));
TLP_PARSER.declareField((p, v, c) -> {
String analyzerName = p.text();
Analyzer analyzer = c.mapperService.analysisService().analyzer(analyzerName);
if (analyzer == null) {
throw new IllegalArgumentException("Analyzer [" + analyzerName + "] doesn't exists");
}
v.setAnalyzer(analyzer);
}, new ParseField("analyzer"), ObjectParser.ValueType.STRING);
TLP_PARSER.declareString(SuggestionSearchContext.SuggestionContext::setField, new ParseField("analyzer"));
TLP_PARSER.declareInt(SuggestionSearchContext.SuggestionContext::setSize, new ParseField("size"));
TLP_PARSER.declareInt(SuggestionSearchContext.SuggestionContext::setShardSize, new ParseField("size"));
TLP_PARSER.declareField((p, v, c) -> {
// Copy the current structure. We will parse, once the mapping is provided
XContentBuilder builder = XContentFactory.contentBuilder(p.contentType());
builder.copyCurrentStructure(p);
BytesReference bytes = builder.bytes();
c.contextParser = XContentFactory.xContent(bytes).createParser(bytes);
p.skipChildren();
}, new ParseField("contexts", "context"), ObjectParser.ValueType.OBJECT); // context is deprecated
}
private static class ContextAndSuggest {
XContentParser contextParser;
final MapperService mapperService;
ContextAndSuggest(MapperService mapperService) {
this.mapperService = mapperService;
}
}
private final CompletionSuggester completionSuggester;
public CompletionSuggestParser(CompletionSuggester completionSuggester) {
@ -79,59 +131,12 @@ public class CompletionSuggestParser implements SuggestContextParser {
}
@Override
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, HasContextAndHeaders headersContext) throws IOException {
XContentParser.Token token;
ParseFieldMatcher parseFieldMatcher = mapperService.getIndexSettings().getParseFieldMatcher();
String fieldName = null;
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService,
HasContextAndHeaders headersContext) throws IOException {
final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester, mapperService, fieldDataService);
XContentParser contextParser = null;
CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptions = null;
CompletionSuggestionBuilder.RegexOptionsBuilder regexOptions = null;
final Set<String> payloadFields = new HashSet<>(1);
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (!parseSuggestContext(parser, mapperService, fieldName, suggestion, parseFieldMatcher)) {
if (token == XContentParser.Token.VALUE_BOOLEAN && "fuzzy".equals(fieldName)) {
if (parser.booleanValue()) {
fuzzyOptions = new CompletionSuggestionBuilder.FuzzyOptionsBuilder();
}
} else if (token == XContentParser.Token.VALUE_STRING && "payload".equals(fieldName)) {
payloadFields.add(parser.text());
}
}
} else if (token == XContentParser.Token.START_OBJECT) {
if ("fuzzy".equals(fieldName)) {
fuzzyOptions = FUZZY_PARSER.parse(parser);
} else if ("contexts".equals(fieldName) || "context".equals(fieldName)) {
// Copy the current structure. We will parse, once the mapping is provided
XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
builder.copyCurrentStructure(parser);
BytesReference bytes = builder.bytes();
contextParser = XContentFactory.xContent(bytes).createParser(bytes);
} else if ("regex".equals(fieldName)) {
regexOptions = REGEXP_PARSER.parse(parser);
} else {
throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]");
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("payload".equals(fieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.VALUE_STRING) {
payloadFields.add(parser.text());
} else {
throw new IllegalArgumentException("suggester [completion] expected string values in [payload] array");
}
}
} else {
throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]");
}
} else {
throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]");
}
}
final ContextAndSuggest contextAndSuggest = new ContextAndSuggest(mapperService);
TLP_PARSER.parse(parser, suggestion, contextAndSuggest);
final XContentParser contextParser = contextAndSuggest.contextParser;
MappedFieldType mappedFieldType = mapperService.smartNameFieldType(suggestion.getField());
if (mappedFieldType == null) {
throw new ElasticsearchException("Field [" + suggestion.getField() + "] is not a completion suggest field");
@ -158,34 +163,13 @@ public class CompletionSuggestParser implements SuggestContextParser {
contextParser.close();
}
suggestion.setFieldType(type);
suggestion.setFuzzyOptionsBuilder(fuzzyOptions);
suggestion.setRegexOptionsBuilder(regexOptions);
suggestion.setQueryContexts(queryContexts);
suggestion.setPayloadFields(payloadFields);
return suggestion;
} else {
throw new IllegalArgumentException("Field [" + suggestion.getField() + "] is not a completion suggest field");
}
}
private static ObjectParser<CompletionSuggestionBuilder.RegexOptionsBuilder, Void> REGEXP_PARSER = new ObjectParser<>("regexp", CompletionSuggestionBuilder.RegexOptionsBuilder::new);
private static ObjectParser<CompletionSuggestionBuilder.FuzzyOptionsBuilder, Void> FUZZY_PARSER = new ObjectParser<>("fuzzy", CompletionSuggestionBuilder.FuzzyOptionsBuilder::new);
static {
FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setFuzzyMinLength, new ParseField("min_length"));
FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setMaxDeterminizedStates, new ParseField("max_determinized_states"));
FUZZY_PARSER.declareBoolean(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setUnicodeAware, new ParseField("unicode_aware"));
FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setFuzzyPrefixLength, new ParseField("prefix_length"));
FUZZY_PARSER.declareBoolean(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setTranspositions, new ParseField("transpositions"));
FUZZY_PARSER.declareValue((a, b) -> {
try {
a.setFuzzyPrefixLength(Fuzziness.parse(b).asDistance());
} catch (IOException e) {
throw new ElasticsearchException(e);
}
}, new ParseField("fuzziness"));
REGEXP_PARSER.declareInt(CompletionSuggestionBuilder.RegexOptionsBuilder::setMaxDeterminizedStates, new ParseField("max_determinized_states"));
REGEXP_PARSER.declareStringOrNull(CompletionSuggestionBuilder.RegexOptionsBuilder::setFlags, new ParseField("flags"));
}
}

9
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
View File

@ -28,10 +28,7 @@ import org.elasticsearch.search.suggest.SuggestionSearchContext;
import org.elasticsearch.search.suggest.completion.context.ContextMapping;
import org.elasticsearch.search.suggest.completion.context.ContextMappings;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
/**
*
@ -85,6 +82,10 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest
this.payloadFields = fields;
}
void setPayloadFields(List<String> fields) {
setPayloadFields(new HashSet<String>(fields));
}
Set<String> getPayloadFields() {
return payloadFields;
}

50
core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
View File

@ -55,7 +55,25 @@ public class ObjectParserTests extends ESTestCase {
assertEquals(s.test, "foo");
assertEquals(s.testNumber, 2);
assertEquals(s.ints, Arrays.asList(1, 2, 3, 4));
assertEquals(objectParser.toString(), "ObjectParser{name='foo', fields=[FieldParser{preferred_name=test, supportedTokens=[VALUE_STRING], type=STRING}, FieldParser{preferred_name=test_number, supportedTokens=[VALUE_STRING, VALUE_NUMBER], type=INT}, FieldParser{preferred_name=test_array, supportedTokens=[START_ARRAY], type=INT_ARRAY}, FieldParser{preferred_name=test_array, supportedTokens=[START_ARRAY], type=INT_ARRAY}, FieldParser{preferred_name=test_number, supportedTokens=[VALUE_STRING, VALUE_NUMBER], type=INT}]}");
assertEquals(objectParser.toString(), "ObjectParser{name='foo', fields=[FieldParser{preferred_name=test, supportedTokens=[VALUE_STRING], type=STRING}, FieldParser{preferred_name=test_number, supportedTokens=[VALUE_STRING, VALUE_NUMBER], type=INT}, FieldParser{preferred_name=test_array, supportedTokens=[START_ARRAY, VALUE_STRING, VALUE_NUMBER], type=INT_ARRAY}, FieldParser{preferred_name=test_array, supportedTokens=[START_ARRAY, VALUE_STRING, VALUE_NUMBER], type=INT_ARRAY}, FieldParser{preferred_name=test_number, supportedTokens=[VALUE_STRING, VALUE_NUMBER], type=INT}]}");
}
public void testObjectOrDefault() throws IOException {
XContentParser parser = XContentType.JSON.xContent().createParser("{\"object\" : { \"test\": 2}}");
ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser("foo", StaticTestStruct::new);
objectParser.declareInt(StaticTestStruct::setTest, new ParseField("test"));
objectParser.declareObjectOrDefault(StaticTestStruct::setObject, objectParser, StaticTestStruct::new, new ParseField("object"));
StaticTestStruct s = objectParser.parse(parser);
assertEquals(s.object.test, 2);
parser = XContentType.JSON.xContent().createParser("{\"object\" : false }");
s = objectParser.parse(parser);
assertNull(s.object);
parser = XContentType.JSON.xContent().createParser("{\"object\" : true }");
s = objectParser.parse(parser);
assertNotNull(s.object);
assertEquals(s.object.test, 0);
}
public void testExceptions() throws IOException {
@ -174,15 +192,35 @@ public class ObjectParserTests extends ESTestCase {
XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
builder.startObject();
builder.field("int_field", randomBoolean() ? "1" : 1);
builder.array("int_array_field", randomBoolean() ? "1" : 1);
if (randomBoolean()) {
builder.array("int_array_field", randomBoolean() ? "1" : 1);
} else {
builder.field("int_array_field", randomBoolean() ? "1" : 1);
}
builder.field("double_field", randomBoolean() ? "2.1" : 2.1d);
builder.array("double_array_field", randomBoolean() ? "2.1" : 2.1d);
if (randomBoolean()) {
builder.array("double_array_field", randomBoolean() ? "2.1" : 2.1d);
} else {
builder.field("double_array_field", randomBoolean() ? "2.1" : 2.1d);
}
builder.field("float_field", randomBoolean() ? "3.1" : 3.1f);
builder.array("float_array_field", randomBoolean() ? "3.1" : 3.1);
if (randomBoolean()) {
builder.array("float_array_field", randomBoolean() ? "3.1" : 3.1);
} else {
builder.field("float_array_field", randomBoolean() ? "3.1" : 3.1);
}
builder.field("long_field", randomBoolean() ? "4" : 4);
builder.array("long_array_field", randomBoolean() ? "4" : 4);
if (randomBoolean()) {
builder.array("long_array_field", randomBoolean() ? "4" : 4);
} else {
builder.field("long_array_field", randomBoolean() ? "4" : 4);
}
builder.field("string_field", "5");
builder.array("string_array_field", "5");
if (randomBoolean()) {
builder.array("string_array_field", "5");
} else {
builder.field("string_array_field", "5");
}
boolean nullValue = randomBoolean();
builder.field("boolean_field", nullValue);
builder.field("string_or_null", nullValue ? null : "5");

6
core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
View File

@ -215,7 +215,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
SuggestResponse suggestResponse = client().suggest(request).get();
assertThat(suggestResponse.getSuccessfulShards(), equalTo(0));
for (ShardOperationFailedException exception : suggestResponse.getShardFailures()) {
assertThat(exception.reason(), containsString("expected string values in [payload] array"));
assertThat(exception.reason(), containsString("ParsingException[[completion] failed to parse field [payload]]; nested: IllegalStateException[expected value but got [START_OBJECT]]"));
}
}
@ -276,12 +276,12 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
assertThat(options.get(1).getScore(), equalTo(1f));
Map<String, List<Object>> firstPayload = options.get(0).getPayload();
assertThat(firstPayload.keySet(), contains("title", "count"));
assertThat(firstPayload.keySet(), containsInAnyOrder("title", "count"));
assertThat((String) firstPayload.get("title").get(0), equalTo("title2"));
assertThat((long) firstPayload.get("count").get(0), equalTo(2l));
Map<String, List<Object>> secondPayload = options.get(1).getPayload();
assertThat(secondPayload.keySet(), contains("title", "count"));
assertThat(secondPayload.keySet(), containsInAnyOrder("title", "count"));
assertThat((String) secondPayload.get("title").get(0), equalTo("title1"));
assertThat((long) secondPayload.get("count").get(0), equalTo(1l));
}

1
core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java
View File

@ -199,7 +199,6 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
.categoryContexts("cat",
new CategoryQueryContext("cat0", 3),
new CategoryQueryContext("cat1"));
assertSuggestions("foo", prefix, "suggestion8", "suggestion6", "suggestion4", "suggestion9", "suggestion2");
}