honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Completion Suggester: Allow payload to be a value 

closes #3550
This commit is contained in:
Shay Banon 2013-08-21 15:33:52 +02:00
parent 210683d70b
commit 25d28f8afa
7 changed files with 180 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
View File

@ -272,7 +272,27 @@ public class JsonXContentGenerator implements XContentGenerator {
}
@Override
public void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException {
public final void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException {
XContentType contentType = XContentFactory.xContentType(content);
if (contentType != null) {
writeObjectRaw(fieldName, content, bos);
} else {
writeFieldName(fieldName);
// we could potentially optimize this to not rely on exception logic...
String sValue = content.toUtf8();
try {
writeNumber(Long.parseLong(sValue));
} catch (NumberFormatException e) {
try {
writeNumber(Double.parseDouble(sValue));
} catch (NumberFormatException e1) {
writeString(sValue);
}
}
}
}
protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException {
generator.writeRaw(", \"");
generator.writeRaw(fieldName);
generator.writeRaw("\" : ");

2
src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java
View File

@ -68,7 +68,7 @@ public class SmileXContentGenerator extends JsonXContentGenerator {
}
@Override
public void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException {
protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException {
writeFieldName(fieldName);
SmileParser parser;
if (content.hasArray()) {

2
src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java
View File

@ -68,7 +68,7 @@ public class YamlXContentGenerator extends JsonXContentGenerator {
}
@Override
public void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException {
protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException {
writeFieldName(fieldName);
YAMLParser parser;
if (content.hasArray()) {

26
src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
View File

@ -74,7 +74,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
public static final String TYPE = "type";
}
public static class Builder extends AbstractFieldMapper.OpenBuilder<Builder, CompletionFieldMapper> {
public static class Builder extends AbstractFieldMapper.OpenBuilder<Builder, CompletionFieldMapper> {
private NamedAnalyzer searchAnalyzer;
private NamedAnalyzer indexAnalyzer;
@ -175,7 +175,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
this.preservePositionIncrements = preservePositionIncrements;
}
@Override
public PostingsFormatProvider postingsFormatProvider() {
return this.completionPostingsFormatProvider;
@ -206,8 +206,10 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
XContentBuilder payloadBuilder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser);
payload = payloadBuilder.bytes().toBytesRef();
payloadBuilder.close();
} else if (token.isValue()) {
payload = parser.bytesOrNull();
} else {
throw new MapperException("Payload must be an object");
throw new MapperException("payload doesn't support type " + token);
}
} else if (token == XContentParser.Token.VALUE_STRING) {
if ("output".equals(currentFieldName)) {
@ -232,7 +234,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
}
}
}
payload = payload == null ? EMPTY: payload;
payload = payload == null ? EMPTY : payload;
if (surfaceForm == null) { // no surface form use the input
for (String input : inputs) {
BytesRef suggestPayload = analyzingSuggestLookupProvider.buildPayload(new BytesRef(
@ -247,7 +249,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
}
}
}
public Field getCompletionField(String input, BytesRef payload) {
return new SuggestField(names().fullName(), input, this.fieldType, payload, analyzingSuggestLookupProvider);
}
@ -274,21 +276,21 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
}
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(name())
.field(Fields.TYPE, CONTENT_TYPE);
.field(Fields.TYPE, CONTENT_TYPE);
if (indexAnalyzer.name().equals(searchAnalyzer.name())) {
builder.field(Fields.ANALYZER, indexAnalyzer.name());
} else {
builder.field(Fields.INDEX_ANALYZER, indexAnalyzer.name())
.field(Fields.SEARCH_ANALYZER, searchAnalyzer.name());
.field(Fields.SEARCH_ANALYZER, searchAnalyzer.name());
}
builder.field(Fields.PAYLOADS, this.payloads)
.field(Fields.PRESERVE_SEPARATORS, this.preserveSeparators)
.field(Fields.PRESERVE_POSITION_INCREMENTS, this.preservePositionIncrements)
.endObject();
.field(Fields.PRESERVE_SEPARATORS, this.preserveSeparators)
.field(Fields.PRESERVE_POSITION_INCREMENTS, this.preservePositionIncrements)
.endObject();
return builder;
}
@ -303,7 +305,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
protected String contentType() {
return CONTENT_TYPE;
}
@Override
public FieldType defaultFieldType() {

24
src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
View File

@ -23,9 +23,11 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.search.suggest.Suggest;
import java.io.IOException;
import java.util.Map;
/**
*
@ -69,11 +71,11 @@ public class CompletionSuggestion extends Suggest.Suggestion<CompletionSuggestio
public static class Option extends org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option {
private BytesReference payload;
public Option(Text text, float score,BytesReference payload) {
public Option(Text text, float score, BytesReference payload) {
super(text, score);
this.payload = payload;
}
protected Option() {
super();
@ -86,7 +88,23 @@ public class CompletionSuggestion extends Suggest.Suggestion<CompletionSuggestio
public BytesReference getPayload() {
return payload;
}
public String getPayloadAsString() {
return payload.toUtf8();
}
public long getPayloadAsLong() {
return Long.parseLong(payload.toUtf8());
}
public double getPayloadAsDouble() {
return Double.parseDouble(payload.toUtf8());
}
public Map<String, Object> getPayloadAsMap() {
return XContentHelper.convertToMap(payload, false).v2();
}
public void setScore(float score) {
super.setScore(score);
}

114
src/test/java/org/elasticsearch/test/integration/search/suggest/CompletionSuggestSearchTests.java
View File

@ -28,7 +28,6 @@ import org.elasticsearch.action.suggest.SuggestResponse;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.mapper.MapperException;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.completion.CompletionStats;
@ -57,14 +56,14 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
private static final String INDEX = "test";
private static final String TYPE = "testType";
private static final String FIELD = "testField";
@Test
public void testSimple() throws Exception{
public void testSimple() throws Exception {
createIndexAndMapping();
String[][] input = {{"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"},
{"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly" },
{"The Prodigy"}, {"The Prodigy"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"},
{"Turbonegro"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; // work with frequencies
{"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"},
{"The Prodigy"}, {"The Prodigy"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"},
{"Turbonegro"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; // work with frequencies
for (int i = 0; i < input.length; i++) {
client().prepareIndex(INDEX, TYPE, "" + i)
.setSource(jsonBuilder()
@ -86,7 +85,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
public void testBasicPrefixSuggestion() throws Exception {
createIndexAndMapping();
for (int i = 0; i < 2; i++) {
createData(i==0);
createData(i == 0);
assertSuggestions("f", "Firestarter - The Prodigy", "Foo Fighters", "Generator - Foo Fighters", "Learn to Fly - Foo Fighters");
assertSuggestions("ge", "Generator - Foo Fighters", "Get it on - Turbonegro");
assertSuggestions("ge", "Generator - Foo Fighters", "Get it on - Turbonegro");
@ -155,7 +154,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertThat(prefixOption.getPayload(), is(notNullValue()));
// parse JSON
Map<String, Object> jsonMap = JsonXContent.jsonXContent.createParser(prefixOption.getPayload()).mapAndClose();
Map<String, Object> jsonMap = prefixOption.getPayloadAsMap();
assertThat(jsonMap.size(), is(2));
assertThat(jsonMap.get("foo").toString(), is("bar"));
assertThat(jsonMap.get("test"), is(instanceOf(List.class)));
@ -163,6 +162,60 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertThat(listValues, hasItems("spam", "eggs"));
}
@Test
public void testPayloadAsNumeric() throws Exception {
createIndexAndMapping();
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD)
.startArray("input").value("Foo Fighters").endArray()
.field("output", "Boo Fighters")
.field("payload", 1)
.endObject().endObject()
).get();
refresh();
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10)
).execute().actionGet();
assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters");
Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0);
assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class)));
CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option;
assertThat(prefixOption.getPayload(), is(notNullValue()));
assertThat(prefixOption.getPayloadAsLong(), equalTo(1l));
}
@Test
public void testPayloadAsString() throws Exception {
createIndexAndMapping();
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD)
.startArray("input").value("Foo Fighters").endArray()
.field("output", "Boo Fighters")
.field("payload", "test")
.endObject().endObject()
).get();
refresh();
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10)
).execute().actionGet();
assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters");
Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0);
assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class)));
CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option;
assertThat(prefixOption.getPayload(), is(notNullValue()));
assertThat(prefixOption.getPayloadAsString(), equalTo("test"));
}
@Test(expected = MapperException.class)
public void testThatExceptionIsThrownWhenPayloadsAreDisabledButInIndexRequest() throws Exception {
createIndexAndMapping("simple", "simple", false, false, true);
@ -176,19 +229,6 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
).get();
}
@Test(expected = MapperException.class)
public void testThatIndexingNonObjectAsPayloadThrowsException() throws Exception {
createIndexAndMapping();
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD)
.startArray("input").value("Foo Fighters").endArray()
.field("output", "Boo Fighters")
.field("payload", "does not work")
.endObject().endObject()
).get();
}
@Test
public void testDisabledPreserveSeperators() throws Exception {
createIndexAndMapping("simple", "simple", true, false, true);
@ -251,7 +291,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
}
@Test
public void testThatShortSyntaxIsWorking() throws Exception {
public void testThatShortSyntaxIsWorking() throws Exception {
createIndexAndMapping();
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
@ -307,16 +347,16 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
@Test
public void testThatUpgradeToMultiFieldWorks() throws Exception {
Settings.Builder settingsBuilder = createDefaultSettings();
final XContentBuilder mapping = jsonBuilder()
final XContentBuilder mapping = jsonBuilder()
.startObject()
.startObject(TYPE)
.startObject("properties")
.startObject(FIELD)
.field("type", "string")
.endObject()
.endObject()
.startObject("properties")
.startObject(FIELD)
.field("type", "string")
.endObject()
.endObject();
.endObject()
.endObject()
.endObject();
client().admin().indices().prepareCreate(INDEX).addMapping(TYPE, mapping).setSettings(settingsBuilder).get();
ensureYellow();
client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get();
@ -484,7 +524,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
.field("type", "completion").field("analyzer", "simple")
.endObject()
.endObject().endObject().endObject())
.get();
.get();
assertThat(putMappingResponse.isAcknowledged(), is(true));
// Index two entities
@ -510,7 +550,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertThat(regexSizeInBytes, is(totalSizeInBytes));
}
public void assertSuggestions(String suggestion, String ... suggestions) {
public void assertSuggestions(String suggestion, String... suggestions) {
String suggestionName = RandomStrings.randomAsciiOfLength(new Random(), 10);
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionBuilder(suggestionName).field(FIELD).text(suggestion).size(10)
@ -519,7 +559,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertSuggestions(suggestResponse, suggestionName, suggestions);
}
public void assertSuggestionsNotInOrder(String suggestString, String ... suggestions) {
public void assertSuggestionsNotInOrder(String suggestString, String... suggestions) {
String suggestionName = RandomStrings.randomAsciiOfLength(new Random(), 10);
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionBuilder(suggestionName).field(FIELD).text(suggestString).size(10)
@ -544,7 +584,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertThat(assertMsg, options.size(), is(suggestions.length));
if (suggestionOrderStrict) {
for (int i = 0; i < suggestions.length; i++) {
String errMsg = String.format(Locale.ROOT, "Expected elem %s in list %s to be [%s] score: %s", i, suggestionList, suggestions[i], options.get(i).getScore());
String errMsg = String.format(Locale.ROOT, "Expected elem %s in list %s to be [%s] score: %s", i, suggestionList, suggestions[i], options.get(i).getScore());
assertThat(errMsg, options.get(i).getText().toString(), is(suggestions[i]));
}
} else {
@ -595,12 +635,12 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
private ImmutableSettings.Builder createDefaultSettings() {
int randomShardNumber = between(1, 5);
int randomReplicaNumber = between(0, numberOfNodes()-1);
int randomReplicaNumber = between(0, numberOfNodes() - 1);
return settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, randomShardNumber).put(SETTING_NUMBER_OF_REPLICAS, randomReplicaNumber);
}
private void createData(boolean optimize) throws IOException, InterruptedException, ExecutionException {
String[][] input = {{"Foo Fighters"}, {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly" }, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}};
String[][] input = {{"Foo Fighters"}, {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}};
String[] surface = {"Foo Fighters", "Generator - Foo Fighters", "Learn to Fly - Foo Fighters", "The Prodigy", "Firestarter - The Prodigy", "Turbonegro", "Get it on - Turbonegro"};
int[] weight = {10, 9, 8, 12, 11, 6, 7};
IndexRequestBuilder[] builders = new IndexRequestBuilder[input.length];
@ -609,7 +649,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
.setSource(jsonBuilder()
.startObject().startObject(FIELD)
.startArray("input").value(input[i]).endArray()
.field("output",surface[i])
.field("output", surface[i])
.startObject("payload").field("id", i).endObject()
.field("weight", 1) // WE FORCEFULLY INDEX A BOGUS WEIGHT
.endObject()
@ -623,7 +663,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
.setSource(jsonBuilder()
.startObject().startObject(FIELD)
.startArray("input").value(input[i]).endArray()
.field("output",surface[i])
.field("output", surface[i])
.startObject("payload").field("id", i).endObject()
.field("weight", weight[i])
.endObject()

45
src/test/java/org/elasticsearch/test/unit/common/xcontent/builder/BuilderRawFieldTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.unit.common.xcontent.builder;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
@ -45,12 +46,23 @@ public class BuilderRawFieldTests {
testRawField(XContentType.SMILE);
}
@Test
public void testYamlRawField() throws IOException {
testRawField(XContentType.YAML);
}
private void testRawField(XContentType type) throws IOException {
XContentBuilder builder = XContentFactory.contentBuilder(type);
builder.startObject();
builder.field("field1", "value1");
builder.rawField("_source", XContentFactory.contentBuilder(type).startObject().field("s_field", "s_value").endObject().bytes());
builder.field("field2", "value2");
builder.rawField("payload_i", new BytesArray(Long.toString(1)));
builder.field("field3", "value3");
builder.rawField("payload_d", new BytesArray(Double.toString(1.1)));
builder.field("field4", "value4");
builder.rawField("payload_s", new BytesArray("test"));
builder.field("field5", "value5");
builder.endObject();
XContentParser parser = XContentFactory.xContent(type).createParser(builder.bytes());
@ -73,6 +85,39 @@ public class BuilderRawFieldTests {
assertThat(parser.currentName(), equalTo("field2"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
assertThat(parser.text(), equalTo("value2"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
assertThat(parser.currentName(), equalTo("payload_i"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER));
assertThat(parser.numberType(), equalTo(XContentParser.NumberType.INT));
assertThat(parser.longValue(), equalTo(1l));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
assertThat(parser.currentName(), equalTo("field3"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
assertThat(parser.text(), equalTo("value3"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
assertThat(parser.currentName(), equalTo("payload_d"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER));
assertThat(parser.numberType(), equalTo(XContentParser.NumberType.DOUBLE));
assertThat(parser.doubleValue(), equalTo(1.1d));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
assertThat(parser.currentName(), equalTo("field4"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
assertThat(parser.text(), equalTo("value4"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
assertThat(parser.currentName(), equalTo("payload_s"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
assertThat(parser.text(), equalTo("test"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
assertThat(parser.currentName(), equalTo("field5"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
assertThat(parser.text(), equalTo("value5"));
assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
}
}