CompletionFieldMapper should use index name instead of full name
Fixes #3669
This commit is contained in:
parent
507b6a6e8c
commit
714aaa40ea
|
@ -175,7 +175,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
|
||||||
private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) {
|
private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) {
|
||||||
NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name);
|
NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name);
|
||||||
if (analyzer == null) {
|
if (analyzer == null) {
|
||||||
throw new ElasticSearchIllegalArgumentException("Can't find default or mapped analyzer with name [" + name +"]");
|
throw new ElasticSearchIllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]");
|
||||||
}
|
}
|
||||||
return analyzer;
|
return analyzer;
|
||||||
}
|
}
|
||||||
|
@ -289,12 +289,12 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
|
||||||
+ "] at position " + i + " is a reserved character");
|
+ "] at position " + i + " is a reserved character");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return new SuggestField(names().fullName(), input, this.fieldType, payload, analyzingSuggestLookupProvider);
|
return new SuggestField(names.indexName(), input, this.fieldType, payload, analyzingSuggestLookupProvider);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static int correctSubStringLen(String input, int len) {
|
public static int correctSubStringLen(String input, int len) {
|
||||||
if (Character.isHighSurrogate(input.charAt(len-1))) {
|
if (Character.isHighSurrogate(input.charAt(len - 1))) {
|
||||||
assert input.length() >= len+1 && Character.isLowSurrogate(input.charAt(len));
|
assert input.length() >= len + 1 && Character.isLowSurrogate(input.charAt(len));
|
||||||
return len + 1;
|
return len + 1;
|
||||||
}
|
}
|
||||||
return len;
|
return len;
|
||||||
|
|
|
@ -31,8 +31,11 @@ import org.apache.lucene.store.IndexOutput;
|
||||||
import org.apache.lucene.util.BytesRef;
|
import org.apache.lucene.util.BytesRef;
|
||||||
import org.apache.lucene.util.IOUtils;
|
import org.apache.lucene.util.IOUtils;
|
||||||
import org.apache.lucene.util.IntsRef;
|
import org.apache.lucene.util.IntsRef;
|
||||||
import org.apache.lucene.util.fst.*;
|
import org.apache.lucene.util.fst.ByteSequenceOutputs;
|
||||||
|
import org.apache.lucene.util.fst.FST;
|
||||||
|
import org.apache.lucene.util.fst.PairOutputs;
|
||||||
import org.apache.lucene.util.fst.PairOutputs.Pair;
|
import org.apache.lucene.util.fst.PairOutputs.Pair;
|
||||||
|
import org.apache.lucene.util.fst.PositiveIntOutputs;
|
||||||
import org.elasticsearch.common.regex.Regex;
|
import org.elasticsearch.common.regex.Regex;
|
||||||
import org.elasticsearch.index.mapper.FieldMapper;
|
import org.elasticsearch.index.mapper.FieldMapper;
|
||||||
import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider;
|
import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider;
|
||||||
|
@ -84,6 +87,7 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
|
||||||
CodecUtil.writeHeader(output, CODEC_NAME, CODEC_VERSION);
|
CodecUtil.writeHeader(output, CODEC_NAME, CODEC_VERSION);
|
||||||
return new FieldsConsumer() {
|
return new FieldsConsumer() {
|
||||||
private Map<FieldInfo, Long> fieldOffsets = new HashMap<FieldInfo, Long>();
|
private Map<FieldInfo, Long> fieldOffsets = new HashMap<FieldInfo, Long>();
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void close() throws IOException {
|
public void close() throws IOException {
|
||||||
try { /*
|
try { /*
|
||||||
|
@ -179,7 +183,7 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
|
||||||
analyzingSuggestLookupProvider.parsePayload(payload, spare);
|
analyzingSuggestLookupProvider.parsePayload(payload, spare);
|
||||||
builder.addSurface(spare.surfaceForm, spare.payload, spare.weight);
|
builder.addSurface(spare.surfaceForm, spare.payload, spare.weight);
|
||||||
// multi fields have the same surface form so we sum up here
|
// multi fields have the same surface form so we sum up here
|
||||||
maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, position+1);
|
maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, position + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -189,7 +193,9 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
|
||||||
public int getMaxAnalyzedPathsForOneInput() {
|
public int getMaxAnalyzedPathsForOneInput() {
|
||||||
return maxAnalyzedPathsForOneInput;
|
return maxAnalyzedPathsForOneInput;
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
|
;
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -225,11 +231,11 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
|
||||||
return new LookupFactory() {
|
return new LookupFactory() {
|
||||||
@Override
|
@Override
|
||||||
public Lookup getLookup(FieldMapper<?> mapper, CompletionSuggestionContext suggestionContext) {
|
public Lookup getLookup(FieldMapper<?> mapper, CompletionSuggestionContext suggestionContext) {
|
||||||
AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(mapper.names().fullName());
|
AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(mapper.names().indexName());
|
||||||
if (analyzingSuggestHolder == null) {
|
if (analyzingSuggestHolder == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
int flags = analyzingSuggestHolder.preserveSep? XAnalyzingSuggester.PRESERVE_SEP : 0;
|
int flags = analyzingSuggestHolder.preserveSep ? XAnalyzingSuggester.PRESERVE_SEP : 0;
|
||||||
|
|
||||||
XAnalyzingSuggester suggester;
|
XAnalyzingSuggester suggester;
|
||||||
if (suggestionContext.isFuzzy()) {
|
if (suggestionContext.isFuzzy()) {
|
||||||
|
@ -251,14 +257,14 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public CompletionStats stats(String ... fields) {
|
public CompletionStats stats(String... fields) {
|
||||||
long sizeInBytes = 0;
|
long sizeInBytes = 0;
|
||||||
TObjectLongHashMap<String> completionFields = null;
|
TObjectLongHashMap<String> completionFields = null;
|
||||||
if (fields != null && fields.length > 0) {
|
if (fields != null && fields.length > 0) {
|
||||||
completionFields = new TObjectLongHashMap<String>(fields.length);
|
completionFields = new TObjectLongHashMap<String>(fields.length);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (Map.Entry<String, AnalyzingSuggestHolder> entry: lookupMap.entrySet()) {
|
for (Map.Entry<String, AnalyzingSuggestHolder> entry : lookupMap.entrySet()) {
|
||||||
sizeInBytes += entry.getValue().fst.sizeInBytes();
|
sizeInBytes += entry.getValue().fst.sizeInBytes();
|
||||||
if (fields == null || fields.length == 0) {
|
if (fields == null || fields.length == 0) {
|
||||||
continue;
|
continue;
|
||||||
|
|
|
@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest;
|
||||||
|
|
||||||
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
|
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import org.elasticsearch.AbstractSharedClusterTest;
|
||||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
|
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
|
||||||
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
|
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
|
||||||
import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
|
import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
|
||||||
|
@ -39,7 +40,6 @@ import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
|
||||||
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
|
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
|
||||||
import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder;
|
import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder;
|
||||||
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
|
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
|
||||||
import org.elasticsearch.AbstractSharedClusterTest;
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -370,6 +370,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
.startObject(TYPE).startObject("properties")
|
.startObject(TYPE).startObject("properties")
|
||||||
.startObject(FIELD)
|
.startObject(FIELD)
|
||||||
.field("type", "multi_field")
|
.field("type", "multi_field")
|
||||||
|
.field("path", "just_name")
|
||||||
.startObject("fields")
|
.startObject("fields")
|
||||||
.startObject(FIELD).field("type", "string").endObject()
|
.startObject(FIELD).field("type", "string").endObject()
|
||||||
.startObject("suggest").field("type", "completion").field("index_analyzer", "simple").field("search_analyzer", "simple").endObject()
|
.startObject("suggest").field("type", "completion").field("index_analyzer", "simple").field("search_analyzer", "simple").endObject()
|
||||||
|
@ -381,7 +382,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
assertThat(putMappingResponse.isAcknowledged(), is(true));
|
assertThat(putMappingResponse.isAcknowledged(), is(true));
|
||||||
|
|
||||||
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
|
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
|
||||||
new CompletionSuggestionBuilder("suggs").field(FIELD + ".suggest").text("f").size(10)
|
new CompletionSuggestionBuilder("suggs").field("suggest").text("f").size(10)
|
||||||
).execute().actionGet();
|
).execute().actionGet();
|
||||||
assertSuggestions(suggestResponse, "suggs");
|
assertSuggestions(suggestResponse, "suggs");
|
||||||
|
|
||||||
|
@ -389,7 +390,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
waitForRelocation(ClusterHealthStatus.GREEN);
|
waitForRelocation(ClusterHealthStatus.GREEN);
|
||||||
|
|
||||||
SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion(
|
SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion(
|
||||||
new CompletionSuggestionBuilder("suggs").field(FIELD + ".suggest").text("f").size(10)
|
new CompletionSuggestionBuilder("suggs").field("suggest").text("f").size(10)
|
||||||
).execute().actionGet();
|
).execute().actionGet();
|
||||||
assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters");
|
assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters");
|
||||||
}
|
}
|
||||||
|
@ -696,7 +697,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
).get();
|
).get();
|
||||||
client().prepareIndex(INDEX, TYPE, "2").setSource(jsonBuilder()
|
client().prepareIndex(INDEX, TYPE, "2").setSource(jsonBuilder()
|
||||||
.startObject()
|
.startObject()
|
||||||
.field("somefield", "somevalue")
|
.field("somefield", "somevalue")
|
||||||
.endObject()
|
.endObject()
|
||||||
).get(); // we have 2 docs in a segment...
|
).get(); // we have 2 docs in a segment...
|
||||||
OptimizeResponse actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).setRefresh(true).execute().actionGet();
|
OptimizeResponse actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).setRefresh(true).execute().actionGet();
|
||||||
|
@ -704,7 +705,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
// update the first one and then merge.. the target segment will have no value in FIELD
|
// update the first one and then merge.. the target segment will have no value in FIELD
|
||||||
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
|
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
|
||||||
.startObject()
|
.startObject()
|
||||||
.field("somefield", "somevalue")
|
.field("somefield", "somevalue")
|
||||||
.endObject()
|
.endObject()
|
||||||
).get();
|
).get();
|
||||||
actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).setRefresh(true).execute().actionGet();
|
actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).setRefresh(true).execute().actionGet();
|
||||||
|
@ -712,7 +713,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
|
|
||||||
assertSuggestions("b");
|
assertSuggestions("b");
|
||||||
assertThat(2l, equalTo(client().prepareCount(INDEX).get().getCount()));
|
assertThat(2l, equalTo(client().prepareCount(INDEX).get().getCount()));
|
||||||
for(IndexShardSegments seg : client().admin().indices().prepareSegments().get().getIndices().get(INDEX)) {
|
for (IndexShardSegments seg : client().admin().indices().prepareSegments().get().getIndices().get(INDEX)) {
|
||||||
ShardSegments[] shards = seg.getShards();
|
ShardSegments[] shards = seg.getShards();
|
||||||
for (ShardSegments shardSegments : shards) {
|
for (ShardSegments shardSegments : shards) {
|
||||||
assertThat(1, equalTo(shardSegments.getSegments().size()));
|
assertThat(1, equalTo(shardSegments.getSegments().size()));
|
||||||
|
@ -726,13 +727,13 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
int iters = atLeast(10);
|
int iters = atLeast(10);
|
||||||
for (int i = 0; i < iters; i++) {
|
for (int i = 0; i < iters; i++) {
|
||||||
int len = between(3, 50);
|
int len = between(3, 50);
|
||||||
String str = replaceReservedChars(randomRealisticUnicodeOfCodepointLengthBetween(len+1, atLeast(len + 2)), (char)0x01);
|
String str = replaceReservedChars(randomRealisticUnicodeOfCodepointLengthBetween(len + 1, atLeast(len + 2)), (char) 0x01);
|
||||||
ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
|
ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
|
||||||
.startObject(TYPE).startObject("properties")
|
.startObject(TYPE).startObject("properties")
|
||||||
.startObject(FIELD)
|
.startObject(FIELD)
|
||||||
.field("type", "completion")
|
.field("type", "completion")
|
||||||
.field("max_input_len", len)
|
.field("max_input_len", len)
|
||||||
// upgrade mapping each time
|
// upgrade mapping each time
|
||||||
.field("analyzer", "keyword")
|
.field("analyzer", "keyword")
|
||||||
.endObject()
|
.endObject()
|
||||||
.endObject().endObject()
|
.endObject().endObject()
|
||||||
|
@ -748,14 +749,14 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
assertSuggestions(str.substring(0, prefixLen), "foobar");
|
assertSuggestions(str.substring(0, prefixLen), "foobar");
|
||||||
if (len + 1 < str.length()) {
|
if (len + 1 < str.length()) {
|
||||||
assertSuggestions(str.substring(0, CompletionFieldMapper.correctSubStringLen(str,
|
assertSuggestions(str.substring(0, CompletionFieldMapper.correctSubStringLen(str,
|
||||||
len + (Character.isHighSurrogate(str.charAt(len-1)) ? 2 : 1))));
|
len + (Character.isHighSurrogate(str.charAt(len - 1)) ? 2 : 1))));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
// see #3596
|
// see #3596
|
||||||
public void testVeryLongInput() throws IOException {
|
public void testVeryLongInput() throws IOException {
|
||||||
client().admin().indices().prepareCreate(INDEX).get();
|
client().admin().indices().prepareCreate(INDEX).get();
|
||||||
ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
|
ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
|
||||||
.startObject(TYPE).startObject("properties")
|
.startObject(TYPE).startObject("properties")
|
||||||
|
@ -766,7 +767,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
.endObject()));
|
.endObject()));
|
||||||
ensureYellow();
|
ensureYellow();
|
||||||
// can cause stack overflow without the default max_input_len
|
// can cause stack overflow without the default max_input_len
|
||||||
String longString = replaceReservedChars(randomRealisticUnicodeOfLength(atLeast(5000)), (char)0x01);
|
String longString = replaceReservedChars(randomRealisticUnicodeOfLength(atLeast(5000)), (char) 0x01);
|
||||||
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
|
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
|
||||||
.startObject().startObject(FIELD)
|
.startObject().startObject(FIELD)
|
||||||
.startArray("input").value(longString).endArray()
|
.startArray("input").value(longString).endArray()
|
||||||
|
@ -778,7 +779,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
|
|
||||||
// see #3648
|
// see #3648
|
||||||
@Test(expected = MapperParsingException.class)
|
@Test(expected = MapperParsingException.class)
|
||||||
public void testReservedChars() throws IOException {
|
public void testReservedChars() throws IOException {
|
||||||
client().admin().indices().prepareCreate(INDEX).get();
|
client().admin().indices().prepareCreate(INDEX).get();
|
||||||
ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
|
ElasticsearchAssertions.assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
|
||||||
.startObject(TYPE).startObject("properties")
|
.startObject(TYPE).startObject("properties")
|
||||||
|
@ -789,7 +790,7 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
|
||||||
.endObject()));
|
.endObject()));
|
||||||
ensureYellow();
|
ensureYellow();
|
||||||
// can cause stack overflow without the default max_input_len
|
// can cause stack overflow without the default max_input_len
|
||||||
String string = "foo" + (char)0x00 + "bar";
|
String string = "foo" + (char) 0x00 + "bar";
|
||||||
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
|
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
|
||||||
.startObject().startObject(FIELD)
|
.startObject().startObject(FIELD)
|
||||||
.startArray("input").value(string).endArray()
|
.startArray("input").value(string).endArray()
|
||||||
|
|
Loading…
Reference in New Issue