CompletionFieldMapper should use index name instead of full name

Fixes #3669
This commit is contained in:
Igor Motov 2013-09-11 14:40:07 -04:00
parent 507b6a6e8c
commit 714aaa40ea
3 changed files with 53 additions and 46 deletions

View File

@ -175,7 +175,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) { private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) {
NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name); NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name);
if (analyzer == null) { if (analyzer == null) {
throw new ElasticSearchIllegalArgumentException("Can't find default or mapped analyzer with name [" + name +"]"); throw new ElasticSearchIllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]");
} }
return analyzer; return analyzer;
} }
@ -289,12 +289,12 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
+ "] at position " + i + " is a reserved character"); + "] at position " + i + " is a reserved character");
} }
} }
return new SuggestField(names().fullName(), input, this.fieldType, payload, analyzingSuggestLookupProvider); return new SuggestField(names.indexName(), input, this.fieldType, payload, analyzingSuggestLookupProvider);
} }
public static int correctSubStringLen(String input, int len) { public static int correctSubStringLen(String input, int len) {
if (Character.isHighSurrogate(input.charAt(len-1))) { if (Character.isHighSurrogate(input.charAt(len - 1))) {
assert input.length() >= len+1 && Character.isLowSurrogate(input.charAt(len)); assert input.length() >= len + 1 && Character.isLowSurrogate(input.charAt(len));
return len + 1; return len + 1;
} }
return len; return len;

View File

@ -31,8 +31,11 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.fst.*; import org.apache.lucene.util.fst.ByteSequenceOutputs;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PairOutputs;
import org.apache.lucene.util.fst.PairOutputs.Pair; import org.apache.lucene.util.fst.PairOutputs.Pair;
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider; import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider;
@ -84,6 +87,7 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
CodecUtil.writeHeader(output, CODEC_NAME, CODEC_VERSION); CodecUtil.writeHeader(output, CODEC_NAME, CODEC_VERSION);
return new FieldsConsumer() { return new FieldsConsumer() {
private Map<FieldInfo, Long> fieldOffsets = new HashMap<FieldInfo, Long>(); private Map<FieldInfo, Long> fieldOffsets = new HashMap<FieldInfo, Long>();
@Override @Override
public void close() throws IOException { public void close() throws IOException {
try { /* try { /*
@ -179,7 +183,7 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
analyzingSuggestLookupProvider.parsePayload(payload, spare); analyzingSuggestLookupProvider.parsePayload(payload, spare);
builder.addSurface(spare.surfaceForm, spare.payload, spare.weight); builder.addSurface(spare.surfaceForm, spare.payload, spare.weight);
// multi fields have the same surface form so we sum up here // multi fields have the same surface form so we sum up here
maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, position+1); maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, position + 1);
} }
@Override @Override
@ -189,7 +193,9 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
public int getMaxAnalyzedPathsForOneInput() { public int getMaxAnalyzedPathsForOneInput() {
return maxAnalyzedPathsForOneInput; return maxAnalyzedPathsForOneInput;
} }
}; }
;
@Override @Override
@ -225,11 +231,11 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
return new LookupFactory() { return new LookupFactory() {
@Override @Override
public Lookup getLookup(FieldMapper<?> mapper, CompletionSuggestionContext suggestionContext) { public Lookup getLookup(FieldMapper<?> mapper, CompletionSuggestionContext suggestionContext) {
AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(mapper.names().fullName()); AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(mapper.names().indexName());
if (analyzingSuggestHolder == null) { if (analyzingSuggestHolder == null) {
return null; return null;
} }
int flags = analyzingSuggestHolder.preserveSep? XAnalyzingSuggester.PRESERVE_SEP : 0; int flags = analyzingSuggestHolder.preserveSep ? XAnalyzingSuggester.PRESERVE_SEP : 0;
XAnalyzingSuggester suggester; XAnalyzingSuggester suggester;
if (suggestionContext.isFuzzy()) { if (suggestionContext.isFuzzy()) {
@ -251,14 +257,14 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
} }
@Override @Override
public CompletionStats stats(String ... fields) { public CompletionStats stats(String... fields) {
long sizeInBytes = 0; long sizeInBytes = 0;
TObjectLongHashMap<String> completionFields = null; TObjectLongHashMap<String> completionFields = null;
if (fields != null && fields.length > 0) { if (fields != null && fields.length > 0) {
completionFields = new TObjectLongHashMap<String>(fields.length); completionFields = new TObjectLongHashMap<String>(fields.length);
} }
for (Map.Entry<String, AnalyzingSuggestHolder> entry: lookupMap.entrySet()) { for (Map.Entry<String, AnalyzingSuggestHolder> entry : lookupMap.entrySet()) {
sizeInBytes += entry.getValue().fst.sizeInBytes(); sizeInBytes += entry.getValue().fst.sizeInBytes();
if (fields == null || fields.length == 0) { if (fields == null || fields.length == 0) {
continue; continue;

View File

@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest;
import com.carrotsearch.randomizedtesting.generators.RandomStrings; import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.elasticsearch.AbstractSharedClusterTest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
@ -39,7 +40,6 @@ import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder; import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder;
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
import org.elasticsearch.AbstractSharedClusterTest;
import org.junit.Test; import org.junit.Test;
import java.io.IOException; import java.io.IOException;
@ -370,6 +370,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
.startObject(TYPE).startObject("properties") .startObject(TYPE).startObject("properties")
.startObject(FIELD) .startObject(FIELD)
.field("type", "multi_field") .field("type", "multi_field")
.field("path", "just_name")
.startObject("fields") .startObject("fields")
.startObject(FIELD).field("type", "string").endObject() .startObject(FIELD).field("type", "string").endObject()
.startObject("suggest").field("type", "completion").field("index_analyzer", "simple").field("search_analyzer", "simple").endObject() .startObject("suggest").field("type", "completion").field("index_analyzer", "simple").field("search_analyzer", "simple").endObject()
@ -381,7 +382,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertThat(putMappingResponse.isAcknowledged(), is(true)); assertThat(putMappingResponse.isAcknowledged(), is(true));
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionBuilder("suggs").field(FIELD + ".suggest").text("f").size(10) new CompletionSuggestionBuilder("suggs").field("suggest").text("f").size(10)
).execute().actionGet(); ).execute().actionGet();
assertSuggestions(suggestResponse, "suggs"); assertSuggestions(suggestResponse, "suggs");
@ -389,7 +390,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
waitForRelocation(ClusterHealthStatus.GREEN); waitForRelocation(ClusterHealthStatus.GREEN);
SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion( SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionBuilder("suggs").field(FIELD + ".suggest").text("f").size(10) new CompletionSuggestionBuilder("suggs").field("suggest").text("f").size(10)
).execute().actionGet(); ).execute().actionGet();
assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters"); assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters");
} }
@ -696,7 +697,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
).get(); ).get();
client().prepareIndex(INDEX, TYPE, "2").setSource(jsonBuilder() client().prepareIndex(INDEX, TYPE, "2").setSource(jsonBuilder()
.startObject() .startObject()
.field("somefield", "somevalue") .field("somefield", "somevalue")
.endObject() .endObject()
).get(); // we have 2 docs in a segment... ).get(); // we have 2 docs in a segment...
OptimizeResponse actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).setRefresh(true).execute().actionGet(); OptimizeResponse actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).setRefresh(true).execute().actionGet();
@ -704,7 +705,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
// update the first one and then merge.. the target segment will have no value in FIELD // update the first one and then merge.. the target segment will have no value in FIELD
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject() .startObject()
.field("somefield", "somevalue") .field("somefield", "somevalue")
.endObject() .endObject()
).get(); ).get();
actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).setRefresh(true).execute().actionGet(); actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).setRefresh(true).execute().actionGet();
@ -712,7 +713,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertSuggestions("b"); assertSuggestions("b");
assertThat(2l, equalTo(client().prepareCount(INDEX).get().getCount())); assertThat(2l, equalTo(client().prepareCount(INDEX).get().getCount()));
for(IndexShardSegments seg : client().admin().indices().prepareSegments().get().getIndices().get(INDEX)) { for (IndexShardSegments seg : client().admin().indices().prepareSegments().get().getIndices().get(INDEX)) {
ShardSegments[] shards = seg.getShards(); ShardSegments[] shards = seg.getShards();
for (ShardSegments shardSegments : shards) { for (ShardSegments shardSegments : shards) {
assertThat(1, equalTo(shardSegments.getSegments().size())); assertThat(1, equalTo(shardSegments.getSegments().size()));
@ -726,13 +727,13 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
int iters = atLeast(10); int iters = atLeast(10);
for (int i = 0; i < iters; i++) { for (int i = 0; i < iters; i++) {
int len = between(3, 50); int len = between(3, 50);
String str = replaceReservedChars(randomRealisticUnicodeOfCodepointLengthBetween(len+1, atLeast(len + 2)), (char)0x01); String str = replaceReservedChars(randomRealisticUnicodeOfCodepointLengthBetween(len + 1, atLeast(len + 2)), (char) 0x01);
ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
.startObject(TYPE).startObject("properties") .startObject(TYPE).startObject("properties")
.startObject(FIELD) .startObject(FIELD)
.field("type", "completion") .field("type", "completion")
.field("max_input_len", len) .field("max_input_len", len)
// upgrade mapping each time // upgrade mapping each time
.field("analyzer", "keyword") .field("analyzer", "keyword")
.endObject() .endObject()
.endObject().endObject() .endObject().endObject()
@ -748,14 +749,14 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertSuggestions(str.substring(0, prefixLen), "foobar"); assertSuggestions(str.substring(0, prefixLen), "foobar");
if (len + 1 < str.length()) { if (len + 1 < str.length()) {
assertSuggestions(str.substring(0, CompletionFieldMapper.correctSubStringLen(str, assertSuggestions(str.substring(0, CompletionFieldMapper.correctSubStringLen(str,
len + (Character.isHighSurrogate(str.charAt(len-1)) ? 2 : 1)))); len + (Character.isHighSurrogate(str.charAt(len - 1)) ? 2 : 1))));
} }
} }
} }
@Test @Test
// see #3596 // see #3596
public void testVeryLongInput() throws IOException { public void testVeryLongInput() throws IOException {
client().admin().indices().prepareCreate(INDEX).get(); client().admin().indices().prepareCreate(INDEX).get();
ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
.startObject(TYPE).startObject("properties") .startObject(TYPE).startObject("properties")
@ -766,7 +767,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
.endObject())); .endObject()));
ensureYellow(); ensureYellow();
// can cause stack overflow without the default max_input_len // can cause stack overflow without the default max_input_len
String longString = replaceReservedChars(randomRealisticUnicodeOfLength(atLeast(5000)), (char)0x01); String longString = replaceReservedChars(randomRealisticUnicodeOfLength(atLeast(5000)), (char) 0x01);
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD) .startObject().startObject(FIELD)
.startArray("input").value(longString).endArray() .startArray("input").value(longString).endArray()
@ -778,7 +779,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
// see #3648 // see #3648
@Test(expected = MapperParsingException.class) @Test(expected = MapperParsingException.class)
public void testReservedChars() throws IOException { public void testReservedChars() throws IOException {
client().admin().indices().prepareCreate(INDEX).get(); client().admin().indices().prepareCreate(INDEX).get();
ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
.startObject(TYPE).startObject("properties") .startObject(TYPE).startObject("properties")
@ -789,7 +790,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
.endObject())); .endObject()));
ensureYellow(); ensureYellow();
// can cause stack overflow without the default max_input_len // can cause stack overflow without the default max_input_len
String string = "foo" + (char)0x00 + "bar"; String string = "foo" + (char) 0x00 + "bar";
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD) .startObject().startObject(FIELD)
.startArray("input").value(string).endArray() .startArray("input").value(string).endArray()