honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CompletionFieldMapper should use index name instead of full name 

Fixes #3669
This commit is contained in:
Igor Motov 2013-09-11 14:40:07 -04:00
parent 507b6a6e8c
commit 714aaa40ea
3 changed files with 53 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
View File

@ -175,7 +175,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) { private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) {
NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name); NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name);
if (analyzer == null) { if (analyzer == null) {
throw new ElasticSearchIllegalArgumentException("Can't find default or mapped analyzer with name [" + name +"]"); throw new ElasticSearchIllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]");
} }
return analyzer; return analyzer;
} }
@ -289,12 +289,12 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
+ "] at position " + i + " is a reserved character"); + "] at position " + i + " is a reserved character");
} }
} }
return new SuggestField(names().fullName(), input, this.fieldType, payload, analyzingSuggestLookupProvider); return new SuggestField(names.indexName(), input, this.fieldType, payload, analyzingSuggestLookupProvider);
} }
public static int correctSubStringLen(String input, int len) { public static int correctSubStringLen(String input, int len) {
if (Character.isHighSurrogate(input.charAt(len-1))) { if (Character.isHighSurrogate(input.charAt(len - 1))) {
assert input.length() >= len+1 && Character.isLowSurrogate(input.charAt(len)); assert input.length() >= len + 1 && Character.isLowSurrogate(input.charAt(len));
return len + 1; return len + 1;
} }
return len; return len;

20
src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java
View File

@ -31,8 +31,11 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.fst.*; import org.apache.lucene.util.fst.ByteSequenceOutputs;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PairOutputs;
import org.apache.lucene.util.fst.PairOutputs.Pair; import org.apache.lucene.util.fst.PairOutputs.Pair;
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider; import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider;
@ -84,6 +87,7 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
CodecUtil.writeHeader(output, CODEC_NAME, CODEC_VERSION); CodecUtil.writeHeader(output, CODEC_NAME, CODEC_VERSION);
return new FieldsConsumer() { return new FieldsConsumer() {
private Map<FieldInfo, Long> fieldOffsets = new HashMap<FieldInfo, Long>(); private Map<FieldInfo, Long> fieldOffsets = new HashMap<FieldInfo, Long>();
@Override @Override
public void close() throws IOException { public void close() throws IOException {
try { /* try { /*
@ -179,7 +183,7 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
analyzingSuggestLookupProvider.parsePayload(payload, spare); analyzingSuggestLookupProvider.parsePayload(payload, spare);
builder.addSurface(spare.surfaceForm, spare.payload, spare.weight); builder.addSurface(spare.surfaceForm, spare.payload, spare.weight);
// multi fields have the same surface form so we sum up here // multi fields have the same surface form so we sum up here
maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, position+1); maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, position + 1);
} }
@Override @Override
@ -189,7 +193,9 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
public int getMaxAnalyzedPathsForOneInput() { public int getMaxAnalyzedPathsForOneInput() {
return maxAnalyzedPathsForOneInput; return maxAnalyzedPathsForOneInput;
} }
}; }
;
@Override @Override
@ -225,11 +231,11 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
return new LookupFactory() { return new LookupFactory() {
@Override @Override
public Lookup getLookup(FieldMapper<?> mapper, CompletionSuggestionContext suggestionContext) { public Lookup getLookup(FieldMapper<?> mapper, CompletionSuggestionContext suggestionContext) {
AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(mapper.names().fullName()); AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(mapper.names().indexName());
if (analyzingSuggestHolder == null) { if (analyzingSuggestHolder == null) {
return null; return null;
} }
int flags = analyzingSuggestHolder.preserveSep? XAnalyzingSuggester.PRESERVE_SEP : 0; int flags = analyzingSuggestHolder.preserveSep ? XAnalyzingSuggester.PRESERVE_SEP : 0;
XAnalyzingSuggester suggester; XAnalyzingSuggester suggester;
if (suggestionContext.isFuzzy()) { if (suggestionContext.isFuzzy()) {
@ -251,14 +257,14 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
} }
@Override @Override
public CompletionStats stats(String ... fields) { public CompletionStats stats(String... fields) {
long sizeInBytes = 0; long sizeInBytes = 0;
TObjectLongHashMap<String> completionFields = null; TObjectLongHashMap<String> completionFields = null;
if (fields != null && fields.length > 0) { if (fields != null && fields.length > 0) {
completionFields = new TObjectLongHashMap<String>(fields.length); completionFields = new TObjectLongHashMap<String>(fields.length);
} }
for (Map.Entry<String, AnalyzingSuggestHolder> entry: lookupMap.entrySet()) { for (Map.Entry<String, AnalyzingSuggestHolder> entry : lookupMap.entrySet()) {
sizeInBytes += entry.getValue().fst.sizeInBytes(); sizeInBytes += entry.getValue().fst.sizeInBytes();
if (fields == null || fields.length == 0) { if (fields == null || fields.length == 0) {
continue; continue;

17
src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchTests.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest;
import com.carrotsearch.randomizedtesting.generators.RandomStrings; import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.elasticsearch.AbstractSharedClusterTest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
@ -39,7 +40,6 @@ import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder; import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder;
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
import org.elasticsearch.AbstractSharedClusterTest;
import org.junit.Test; import org.junit.Test;
import java.io.IOException; import java.io.IOException;
@ -370,6 +370,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
.startObject(TYPE).startObject("properties") .startObject(TYPE).startObject("properties")
.startObject(FIELD) .startObject(FIELD)
.field("type", "multi_field") .field("type", "multi_field")
.field("path", "just_name")
.startObject("fields") .startObject("fields")
.startObject(FIELD).field("type", "string").endObject() .startObject(FIELD).field("type", "string").endObject()
.startObject("suggest").field("type", "completion").field("index_analyzer", "simple").field("search_analyzer", "simple").endObject() .startObject("suggest").field("type", "completion").field("index_analyzer", "simple").field("search_analyzer", "simple").endObject()
@ -381,7 +382,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertThat(putMappingResponse.isAcknowledged(), is(true)); assertThat(putMappingResponse.isAcknowledged(), is(true));
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionBuilder("suggs").field(FIELD + ".suggest").text("f").size(10) new CompletionSuggestionBuilder("suggs").field("suggest").text("f").size(10)
).execute().actionGet(); ).execute().actionGet();
assertSuggestions(suggestResponse, "suggs"); assertSuggestions(suggestResponse, "suggs");
@ -389,7 +390,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
waitForRelocation(ClusterHealthStatus.GREEN); waitForRelocation(ClusterHealthStatus.GREEN);
SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion( SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionBuilder("suggs").field(FIELD + ".suggest").text("f").size(10) new CompletionSuggestionBuilder("suggs").field("suggest").text("f").size(10)
).execute().actionGet(); ).execute().actionGet();
assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters"); assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters");
} }
@ -712,7 +713,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertSuggestions("b"); assertSuggestions("b");
assertThat(2l, equalTo(client().prepareCount(INDEX).get().getCount())); assertThat(2l, equalTo(client().prepareCount(INDEX).get().getCount()));
for(IndexShardSegments seg : client().admin().indices().prepareSegments().get().getIndices().get(INDEX)) { for (IndexShardSegments seg : client().admin().indices().prepareSegments().get().getIndices().get(INDEX)) {
ShardSegments[] shards = seg.getShards(); ShardSegments[] shards = seg.getShards();
for (ShardSegments shardSegments : shards) { for (ShardSegments shardSegments : shards) {
assertThat(1, equalTo(shardSegments.getSegments().size())); assertThat(1, equalTo(shardSegments.getSegments().size()));
@ -726,7 +727,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
int iters = atLeast(10); int iters = atLeast(10);
for (int i = 0; i < iters; i++) { for (int i = 0; i < iters; i++) {
int len = between(3, 50); int len = between(3, 50);
String str = replaceReservedChars(randomRealisticUnicodeOfCodepointLengthBetween(len+1, atLeast(len + 2)), (char)0x01); String str = replaceReservedChars(randomRealisticUnicodeOfCodepointLengthBetween(len + 1, atLeast(len + 2)), (char) 0x01);
ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
.startObject(TYPE).startObject("properties") .startObject(TYPE).startObject("properties")
.startObject(FIELD) .startObject(FIELD)
@ -748,7 +749,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertSuggestions(str.substring(0, prefixLen), "foobar"); assertSuggestions(str.substring(0, prefixLen), "foobar");
if (len + 1 < str.length()) { if (len + 1 < str.length()) {
assertSuggestions(str.substring(0, CompletionFieldMapper.correctSubStringLen(str, assertSuggestions(str.substring(0, CompletionFieldMapper.correctSubStringLen(str,
len + (Character.isHighSurrogate(str.charAt(len-1)) ? 2 : 1)))); len + (Character.isHighSurrogate(str.charAt(len - 1)) ? 2 : 1))));
} }
} }
} }
@ -766,7 +767,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
.endObject())); .endObject()));
ensureYellow(); ensureYellow();
// can cause stack overflow without the default max_input_len // can cause stack overflow without the default max_input_len
String longString = replaceReservedChars(randomRealisticUnicodeOfLength(atLeast(5000)), (char)0x01); String longString = replaceReservedChars(randomRealisticUnicodeOfLength(atLeast(5000)), (char) 0x01);
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD) .startObject().startObject(FIELD)
.startArray("input").value(longString).endArray() .startArray("input").value(longString).endArray()
@ -789,7 +790,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
.endObject())); .endObject()));
ensureYellow(); ensureYellow();
// can cause stack overflow without the default max_input_len // can cause stack overflow without the default max_input_len
String string = "foo" + (char)0x00 + "bar"; String string = "foo" + (char) 0x00 + "bar";
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD) .startObject().startObject(FIELD)
.startArray("input").value(string).endArray() .startArray("input").value(string).endArray()