honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move MappedFieldType#getSearchAnalyzer and #getSearchQuoteAnalyzer to TextSearchInfo (#58830) 

Analyzers are specific to text searching, and so should be in TextSearchInfo rather than on
the generic MappedFieldType.

Backport of #58639
This commit is contained in:
Alan Woodward 2020-07-01 14:52:14 +01:00 committed by GitHub
parent d35e8f45da
commit 3ba16e0f39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 240 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java
View File

@ -101,7 +101,6 @@ public class RankFeatureFieldMapper extends FieldMapper {
super(name, true, false, TextSearchInfo.NONE, meta);
this.positiveScoreImpact = positiveScoreImpact;
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
protected RankFeatureFieldType(RankFeatureFieldType ref) {

1
modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java
View File

@ -78,7 +78,6 @@ public class RankFeaturesFieldMapper extends FieldMapper {
public RankFeaturesFieldType(String name, Map<String, String> meta) {
super(name, false, false, TextSearchInfo.NONE, meta);
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
protected RankFeaturesFieldType(RankFeaturesFieldType ref) {

46
modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java
View File

@ -169,10 +169,9 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
@Override
public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) {
SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, similarity, meta);
SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, similarity,
searchAnalyzer, searchQuoteAnalyzer, meta);
ft.setIndexAnalyzer(indexAnalyzer);
ft.setSearchAnalyzer(searchAnalyzer);
ft.setSearchQuoteAnalyzer(searchQuoteAnalyzer);
// set up the prefix field
FieldType prefixft = new FieldType(fieldType);
@ -180,18 +179,20 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
prefixft.setOmitNorms(true);
prefixft.setStored(false);
final String fullName = buildFullName(context);
final PrefixFieldType prefixFieldType = new PrefixFieldType(fullName, prefixft, Defaults.MIN_GRAM, Defaults.MAX_GRAM);
// wrap the root field's index analyzer with shingles and edge ngrams
final SearchAsYouTypeAnalyzer prefixIndexWrapper =
final Analyzer prefixIndexWrapper =
SearchAsYouTypeAnalyzer.withShingleAndPrefix(indexAnalyzer.analyzer(), maxShingleSize);
// wrap the root field's search analyzer with only shingles
final SearchAsYouTypeAnalyzer prefixSearchWrapper =
SearchAsYouTypeAnalyzer.withShingle(searchAnalyzer.analyzer(), maxShingleSize);
final NamedAnalyzer prefixSearchWrapper = new NamedAnalyzer(searchAnalyzer.name(), searchAnalyzer.scope(),
SearchAsYouTypeAnalyzer.withShingle(searchAnalyzer.analyzer(), maxShingleSize));
// don't wrap the root field's search quote analyzer as prefix field doesn't support phrase queries
TextSearchInfo prefixSearchInfo = new TextSearchInfo(prefixft, similarity, prefixSearchWrapper, searchQuoteAnalyzer);
final PrefixFieldType prefixFieldType
= new PrefixFieldType(fullName, prefixSearchInfo, Defaults.MIN_GRAM, Defaults.MAX_GRAM);
prefixFieldType.setIndexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper));
prefixFieldType.setSearchAnalyzer(new NamedAnalyzer(searchAnalyzer.name(), AnalyzerScope.INDEX, prefixSearchWrapper));
final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType);
// set up the shingle fields
final ShingleFieldMapper[] shingleFieldMappers = new ShingleFieldMapper[maxShingleSize - 1];
final ShingleFieldType[] shingleFieldTypes = new ShingleFieldType[maxShingleSize - 1];
@ -200,18 +201,17 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
FieldType shingleft = new FieldType(fieldType);
shingleft.setStored(false);
String fieldName = getShingleFieldName(buildFullName(context), shingleSize);
final ShingleFieldType shingleFieldType = new ShingleFieldType(fieldName, shingleSize, shingleft);
// wrap the root field's index, search, and search quote analyzers with shingles
final SearchAsYouTypeAnalyzer shingleIndexWrapper =
SearchAsYouTypeAnalyzer.withShingle(indexAnalyzer.analyzer(), shingleSize);
final SearchAsYouTypeAnalyzer shingleSearchWrapper =
SearchAsYouTypeAnalyzer.withShingle(searchAnalyzer.analyzer(), shingleSize);
final SearchAsYouTypeAnalyzer shingleSearchQuoteWrapper =
SearchAsYouTypeAnalyzer.withShingle(searchQuoteAnalyzer.analyzer(), shingleSize);
final NamedAnalyzer shingleSearchWrapper = new NamedAnalyzer(searchAnalyzer.name(), searchAnalyzer.scope(),
SearchAsYouTypeAnalyzer.withShingle(searchAnalyzer.analyzer(), shingleSize));
final NamedAnalyzer shingleSearchQuoteWrapper = new NamedAnalyzer(searchQuoteAnalyzer.name(), searchQuoteAnalyzer.scope(),
SearchAsYouTypeAnalyzer.withShingle(searchQuoteAnalyzer.analyzer(), shingleSize));
TextSearchInfo textSearchInfo
= new TextSearchInfo(shingleft, similarity, shingleSearchWrapper, shingleSearchQuoteWrapper);
final ShingleFieldType shingleFieldType = new ShingleFieldType(fieldName, shingleSize, textSearchInfo);
shingleFieldType.setIndexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper));
shingleFieldType.setSearchAnalyzer(new NamedAnalyzer(searchAnalyzer.name(), AnalyzerScope.INDEX, shingleSearchWrapper));
shingleFieldType.setSearchQuoteAnalyzer(
new NamedAnalyzer(searchQuoteAnalyzer.name(), AnalyzerScope.INDEX, shingleSearchQuoteWrapper));
shingleFieldType.setPrefixFieldType(prefixFieldType);
shingleFieldTypes[i] = shingleFieldType;
shingleFieldMappers[i] = new ShingleFieldMapper(shingleft, shingleFieldType);
@ -246,8 +246,10 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
PrefixFieldType prefixField;
ShingleFieldType[] shingleFields = new ShingleFieldType[0];
SearchAsYouTypeFieldType(String name, FieldType fieldType, SimilarityProvider similarity, Map<String, String> meta) {
super(name, fieldType.indexOptions() != IndexOptions.NONE, false, new TextSearchInfo(fieldType, similarity), meta);
SearchAsYouTypeFieldType(String name, FieldType fieldType, SimilarityProvider similarity,
NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer, Map<String, String> meta) {
super(name, fieldType.indexOptions() != IndexOptions.NONE, false,
new TextSearchInfo(fieldType, similarity, searchAnalyzer, searchQuoteAnalyzer), meta);
}
SearchAsYouTypeFieldType(SearchAsYouTypeFieldType other) {
@ -392,8 +394,8 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
final int maxChars;
final String parentField;
PrefixFieldType(String parentField, FieldType fieldType, int minChars, int maxChars) {
super(parentField + PREFIX_FIELD_SUFFIX, true, false, new TextSearchInfo(fieldType, null), Collections.emptyMap());
PrefixFieldType(String parentField, TextSearchInfo textSearchInfo, int minChars, int maxChars) {
super(parentField + PREFIX_FIELD_SUFFIX, true, false, textSearchInfo, Collections.emptyMap());
this.minChars = minChars;
this.maxChars = maxChars;
this.parentField = parentField;
@ -545,8 +547,8 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
final int shingleSize;
PrefixFieldType prefixFieldType;
ShingleFieldType(String name, int shingleSize, FieldType fieldType) {
super(name, true, false, new TextSearchInfo(fieldType, null), Collections.emptyMap());
ShingleFieldType(String name, int shingleSize, TextSearchInfo textSearchInfo) {
super(name, true, false, textSearchInfo, Collections.emptyMap());
this.shingleSize = shingleSize;
}

24
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
View File

@ -669,21 +669,6 @@ public class SearchAsYouTypeFieldMapperTests extends FieldMapperTestCase<SearchA
assertThat(actual, equalTo(expected));
}
// todo are these queries generated for the prefix field right?
{
final Query actual = new MatchPhraseQueryBuilder("a_field._index_prefix", "one two")
.toQuery(queryShardContext);
final Query expected = new MatchNoDocsQuery("Matching no documents because no terms present");
assertThat(actual, equalTo(expected));
}
{
final Query actual = new MatchPhraseQueryBuilder("a_field._index_prefix", "one two three")
.toQuery(queryShardContext);
final Query expected = new TermQuery(new Term("a_field._index_prefix", "one two three"));
assertThat(actual, equalTo(expected));
}
{
expectThrows(IllegalArgumentException.class,
() -> new MatchPhraseQueryBuilder("a_field._index_prefix", "one two three four").toQuery(queryShardContext));
@ -817,7 +802,7 @@ public class SearchAsYouTypeFieldMapperTests extends FieldMapperTestCase<SearchA
PrefixFieldType prefixFieldType) {
assertThat(fieldType.shingleFields.length, equalTo(maxShingleSize-1));
for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.searchAnalyzer())) {
for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) {
assertThat(analyzer.name(), equalTo(analyzerName));
}
int shingleSize = 2;
@ -835,7 +820,7 @@ public class SearchAsYouTypeFieldMapperTests extends FieldMapperTestCase<SearchA
assertThat(fieldType.shingleSize, equalTo(shingleSize));
for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.searchAnalyzer())) {
for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) {
assertThat(analyzer.name(), equalTo(analyzerName));
if (shingleSize > 1) {
final SearchAsYouTypeAnalyzer wrappedAnalyzer = (SearchAsYouTypeAnalyzer) analyzer.analyzer();
@ -849,12 +834,13 @@ public class SearchAsYouTypeFieldMapperTests extends FieldMapperTestCase<SearchA
}
private static void assertPrefixFieldType(PrefixFieldType fieldType, int shingleSize, String analyzerName) {
for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.searchAnalyzer())) {
for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) {
assertThat(analyzer.name(), equalTo(analyzerName));
}
final SearchAsYouTypeAnalyzer wrappedIndexAnalyzer = (SearchAsYouTypeAnalyzer) fieldType.indexAnalyzer().analyzer();
final SearchAsYouTypeAnalyzer wrappedSearchAnalyzer = (SearchAsYouTypeAnalyzer) fieldType.searchAnalyzer().analyzer();
final SearchAsYouTypeAnalyzer wrappedSearchAnalyzer
= (SearchAsYouTypeAnalyzer) fieldType.getTextSearchInfo().getSearchAnalyzer().analyzer();
for (SearchAsYouTypeAnalyzer analyzer : asList(wrappedIndexAnalyzer, wrappedSearchAnalyzer)) {
assertThat(analyzer.shingleSize(), equalTo(shingleSize));
}

16
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java
View File

@ -28,6 +28,7 @@ import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.index.mapper.SearchAsYouTypeFieldMapper.Defaults;
import org.elasticsearch.index.mapper.SearchAsYouTypeFieldMapper.PrefixFieldType;
import org.elasticsearch.index.mapper.SearchAsYouTypeFieldMapper.SearchAsYouTypeFieldType;
@ -51,9 +52,12 @@ public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase<MappedField
@Override
protected SearchAsYouTypeFieldType createDefaultFieldType(String name, Map<String, String> meta) {
final SearchAsYouTypeFieldType fieldType = new SearchAsYouTypeFieldType(name, Defaults.FIELD_TYPE, null, meta);
fieldType.setPrefixField(new PrefixFieldType(NAME, Defaults.FIELD_TYPE, Defaults.MIN_GRAM, Defaults.MAX_GRAM));
fieldType.setShingleFields(new ShingleFieldType[] { new ShingleFieldType(fieldType.name(), 2, Defaults.FIELD_TYPE) });
final SearchAsYouTypeFieldType fieldType
= new SearchAsYouTypeFieldType(name, Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER, meta);
fieldType.setPrefixField(new PrefixFieldType(NAME, TextSearchInfo.SIMPLE_MATCH_ONLY, Defaults.MIN_GRAM, Defaults.MAX_GRAM));
fieldType.setShingleFields(new ShingleFieldType[] {
new ShingleFieldType(fieldType.name(), 2, TextSearchInfo.SIMPLE_MATCH_ONLY)
});
return fieldType;
}
@ -62,7 +66,8 @@ public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase<MappedField
assertThat(fieldType.termQuery("foo", null), equalTo(new TermQuery(new Term(NAME, "foo"))));
SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, null, Collections.emptyMap());
SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, null,
Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER, Collections.emptyMap());
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("foo", null));
assertThat(e.getMessage(), equalTo("Cannot search on field [" + NAME + "] since it is not indexed."));
}
@ -73,7 +78,8 @@ public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase<MappedField
assertThat(fieldType.termsQuery(asList("foo", "bar"), null),
equalTo(new TermInSetQuery(NAME, asList(new BytesRef("foo"), new BytesRef("bar")))));
SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, null, Collections.emptyMap());
SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, null,
Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER, Collections.emptyMap());
final IllegalArgumentException e =
expectThrows(IllegalArgumentException.class, () -> unsearchable.termsQuery(asList("foo", "bar"), null));
assertThat(e.getMessage(), equalTo("Cannot search on field [" + NAME + "] since it is not indexed."));

1
modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java
View File

@ -97,7 +97,6 @@ public final class ParentIdFieldMapper extends FieldMapper {
ParentIdFieldType(String name, boolean eagerGlobalOrdinals, Map<String, String> meta) {
super(name, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
setEagerGlobalOrdinals(eagerGlobalOrdinals);
}

1
modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java
View File

@ -208,7 +208,6 @@ public final class ParentJoinFieldMapper extends FieldMapper {
public JoinFieldType(String name, Map<String, String> meta) {
super(name, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
protected JoinFieldType(JoinFieldType ref) {

2
modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java
View File

@ -282,7 +282,7 @@ public class ChildrenToParentAggregatorTests extends AggregatorTestCase {
when(metaJoinFieldType.getJoinField()).thenReturn("join_field");
when(mapperService.fieldType("_parent_join")).thenReturn(metaJoinFieldType);
DocumentFieldMappers fieldMappers = new DocumentFieldMappers(Collections.singleton(joinFieldMapper),
Collections.emptyList(), null, null, null);
Collections.emptyList(), null);
DocumentMapper mockMapper = mock(DocumentMapper.class);
when(mockMapper.mappers()).thenReturn(fieldMappers);
when(mapperService.documentMapper()).thenReturn(mockMapper);

2
modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java
View File

@ -233,7 +233,7 @@ public class ParentToChildrenAggregatorTests extends AggregatorTestCase {
when(metaJoinFieldType.getJoinField()).thenReturn("join_field");
when(mapperService.fieldType("_parent_join")).thenReturn(metaJoinFieldType);
DocumentFieldMappers fieldMappers = new DocumentFieldMappers(Collections.singleton(joinFieldMapper),
Collections.emptyList(), null, null, null);
Collections.emptyList(), null);
DocumentMapper mockMapper = mock(DocumentMapper.class);
when(mockMapper.mappers()).thenReturn(fieldMappers);
when(mapperService.documentMapper()).thenReturn(mockMapper);

1
plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java
View File

@ -79,7 +79,6 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
public CollationFieldType(String name, boolean isSearchable, boolean hasDocValues, Collator collator, Map<String, String> meta) {
super(name, isSearchable, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
this.collator = collator;
}

58
plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
View File

@ -42,6 +42,7 @@ import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.TextFieldMapper;
import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText.AnnotationToken;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.search.fetch.FetchSubPhase.HitContext;
import java.io.IOException;
@ -77,20 +78,12 @@ public class AnnotatedTextFieldMapper extends FieldMapper {
public static final String CONTENT_TYPE = "annotated_text";
private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1;
public static class Defaults {
public static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
FIELD_TYPE.freeze();
}
}
public static class Builder extends FieldMapper.Builder<Builder> {
public static class Builder extends TextFieldMapper.Builder {
private int positionIncrementGap = POSITION_INCREMENT_GAP_USE_ANALYZER;
public Builder(String name) {
super(name, Defaults.FIELD_TYPE);
super(name);
builder = this;
}
@ -107,30 +100,28 @@ public class AnnotatedTextFieldMapper extends FieldMapper {
if (docValues) {
throw new IllegalArgumentException("[" + CONTENT_TYPE + "] fields do not support doc values");
}
return super.docValues(docValues);
return this;
}
private NamedAnalyzer wrapAnalyzer(NamedAnalyzer in, int positionIncrementGap) {
return new NamedAnalyzer(in.name(), AnalyzerScope.INDEX,
new AnnotationAnalyzerWrapper(in.analyzer()), positionIncrementGap);
}
private AnnotatedTextFieldType buildFieldType(BuilderContext context) {
boolean hasPositions = fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
AnnotatedTextFieldType ft = new AnnotatedTextFieldType(buildFullName(context), hasPositions, meta);
if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
throw new IllegalArgumentException("Cannot set position_increment_gap on field ["
+ name + "] without positions enabled");
}
ft.setIndexAnalyzer(indexAnalyzer, positionIncrementGap);
ft.setSearchAnalyzer(new NamedAnalyzer(searchAnalyzer, positionIncrementGap));
ft.setSearchQuoteAnalyzer(new NamedAnalyzer(searchQuoteAnalyzer, positionIncrementGap));
int posGap;
if (positionIncrementGap == POSITION_INCREMENT_GAP_USE_ANALYZER) {
posGap = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP;
} else {
//Using the analyzer's default BUT need to do the same thing AnalysisRegistry.processAnalyzerFactory
// does to splice in new default of posIncGap=100 by wrapping the analyzer
if (hasPositions) {
int overrideInc = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP;
ft.setIndexAnalyzer(indexAnalyzer, overrideInc);
ft.setSearchAnalyzer(new NamedAnalyzer(searchAnalyzer, overrideInc));
ft.setSearchQuoteAnalyzer(new NamedAnalyzer(searchQuoteAnalyzer,overrideInc));
if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + name()
+ "] without positions enabled");
}
posGap = positionIncrementGap;
}
AnnotatedTextFieldType ft = new AnnotatedTextFieldType(buildFullName(context), fieldType, similarity,
wrapAnalyzer(searchAnalyzer, posGap), wrapAnalyzer(searchQuoteAnalyzer, posGap), meta);
ft.setIndexAnalyzer(indexAnalyzer, posGap);
return ft;
}
@ -147,7 +138,7 @@ public class AnnotatedTextFieldMapper extends FieldMapper {
public static class TypeParser implements Mapper.TypeParser {
@Override
public Mapper.Builder<AnnotatedTextFieldMapper.Builder> parse(
public Mapper.Builder<?> parse(
String fieldName, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
AnnotatedTextFieldMapper.Builder builder = new AnnotatedTextFieldMapper.Builder(fieldName);
builder.indexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer());
@ -521,8 +512,13 @@ public class AnnotatedTextFieldMapper extends FieldMapper {
public static final class AnnotatedTextFieldType extends TextFieldMapper.TextFieldType {
public AnnotatedTextFieldType(String name, boolean hasPositions, Map<String, String> meta) {
super(name, hasPositions, meta);
public AnnotatedTextFieldType(String name, FieldType fieldType, SimilarityProvider similarity,
NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer, Map<String, String> meta) {
super(name, fieldType, similarity, searchAnalyzer, searchQuoteAnalyzer, meta);
}
public AnnotatedTextFieldType(String name, Map<String, String> meta) {
super(name, true, meta);
}
protected AnnotatedTextFieldType(AnnotatedTextFieldType ref) {

2
plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java
View File

@ -34,7 +34,7 @@ import java.util.Map;
public class AnnotatedTextFieldTypeTests extends FieldTypeTestCase<MappedFieldType> {
@Override
protected MappedFieldType createDefaultFieldType(String name, Map<String, String> meta) {
return new AnnotatedTextFieldMapper.AnnotatedTextFieldType(name, true, meta);
return new AnnotatedTextFieldMapper.AnnotatedTextFieldType(name, meta);
}
public void testIntervals() throws IOException {

3
server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.common.lucene;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.document.LatLonDocValuesField;
@ -109,6 +110,8 @@ public class Lucene {
public static final NamedAnalyzer STANDARD_ANALYZER = new NamedAnalyzer("_standard", AnalyzerScope.GLOBAL, new StandardAnalyzer());
public static final NamedAnalyzer KEYWORD_ANALYZER = new NamedAnalyzer("_keyword", AnalyzerScope.GLOBAL, new KeywordAnalyzer());
public static final NamedAnalyzer WHITESPACE_ANALYZER
= new NamedAnalyzer("_whitespace", AnalyzerScope.GLOBAL, new WhitespaceAnalyzer());
public static final ScoreDoc[] EMPTY_SCORE_DOCS = new ScoreDoc[0];

44
server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
View File

@ -37,6 +37,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
@ -194,12 +195,17 @@ public class CompletionFieldMapper extends FieldMapper {
private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS;
private ContextMappings contextMappings = null;
public CompletionFieldType(String name, FieldType luceneFieldType, Map<String, String> meta) {
super(name, true, false, new TextSearchInfo(luceneFieldType, null), meta);
public CompletionFieldType(String name, FieldType luceneFieldType,
NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer, Map<String, String> meta) {
super(name, true, false,
new TextSearchInfo(luceneFieldType, null, searchAnalyzer, searchQuoteAnalyzer), meta);
}
public CompletionFieldType(String name) {
this(name, Defaults.FIELD_TYPE, Collections.emptyMap());
this(name, Defaults.FIELD_TYPE,
new NamedAnalyzer("completion", AnalyzerScope.INDEX, new CompletionAnalyzer(Lucene.STANDARD_ANALYZER)),
new NamedAnalyzer("completion", AnalyzerScope.INDEX, new CompletionAnalyzer(Lucene.STANDARD_ANALYZER)),
Collections.emptyMap());
}
private CompletionFieldType(CompletionFieldType ref) {
@ -232,16 +238,6 @@ public class CompletionFieldMapper extends FieldMapper {
return indexAnalyzer;
}
@Override
public NamedAnalyzer searchAnalyzer() {
final NamedAnalyzer searchAnalyzer = super.searchAnalyzer();
if (searchAnalyzer != null && !(searchAnalyzer.analyzer() instanceof CompletionAnalyzer)) {
return new NamedAnalyzer(searchAnalyzer.name(), AnalyzerScope.INDEX,
new CompletionAnalyzer(searchAnalyzer, preserveSep, preservePositionIncrements));
}
return searchAnalyzer;
}
/**
* @return true if there are one or more context mappings defined
* for this field type
@ -284,7 +280,8 @@ public class CompletionFieldMapper extends FieldMapper {
* Completion prefix query
*/
public CompletionQuery prefixQuery(Object value) {
return new PrefixCompletionQuery(searchAnalyzer().analyzer(), new Term(name(), indexedValueForSearch(value)));
return new PrefixCompletionQuery(getTextSearchInfo().getSearchAnalyzer().analyzer(),
new Term(name(), indexedValueForSearch(value)));
}
/**
@ -300,9 +297,10 @@ public class CompletionFieldMapper extends FieldMapper {
public CompletionQuery fuzzyQuery(String value, Fuzziness fuzziness, int nonFuzzyPrefixLength,
int minFuzzyPrefixLength, int maxExpansions, boolean transpositions,
boolean unicodeAware) {
return new FuzzyCompletionQuery(searchAnalyzer().analyzer(), new Term(name(), indexedValueForSearch(value)), null,
fuzziness.asDistance(), transpositions, nonFuzzyPrefixLength, minFuzzyPrefixLength,
unicodeAware, maxExpansions);
return new FuzzyCompletionQuery(getTextSearchInfo().getSearchAnalyzer().analyzer(),
new Term(name(), indexedValueForSearch(value)), null,
fuzziness.asDistance(), transpositions, nonFuzzyPrefixLength, minFuzzyPrefixLength,
unicodeAware, maxExpansions);
}
@Override
@ -395,13 +393,15 @@ public class CompletionFieldMapper extends FieldMapper {
@Override
public CompletionFieldMapper build(BuilderContext context) {
checkCompletionContextsLimit(context);
CompletionFieldType ft = new CompletionFieldType(buildFullName(context), this.fieldType, meta);
NamedAnalyzer searchAnalyzer = new NamedAnalyzer(this.searchAnalyzer.name(), AnalyzerScope.INDEX,
new CompletionAnalyzer(this.searchAnalyzer, preserveSeparators, preservePositionIncrements));
CompletionFieldType ft
= new CompletionFieldType(buildFullName(context), this.fieldType, searchAnalyzer, searchAnalyzer, meta);
ft.setContextMappings(contextMappings);
ft.setPreservePositionIncrements(preservePositionIncrements);
ft.setPreserveSep(preserveSeparators);
ft.setIndexAnalyzer(indexAnalyzer);
ft.setSearchAnalyzer(searchAnalyzer);
ft.setSearchQuoteAnalyzer(searchQuoteAnalyzer);
return new CompletionFieldMapper(name, this.fieldType, ft,
multiFieldsBuilder.build(this, context), copyTo, maxInputLength);
}
@ -641,8 +641,8 @@ public class CompletionFieldMapper extends FieldMapper {
builder.startObject(simpleName())
.field(Fields.TYPE.getPreferredName(), CONTENT_TYPE);
builder.field(Fields.ANALYZER.getPreferredName(), fieldType().indexAnalyzer().name());
if (fieldType().indexAnalyzer().name().equals(fieldType().searchAnalyzer().name()) == false) {
builder.field(Fields.SEARCH_ANALYZER.getPreferredName(), fieldType().searchAnalyzer().name());
if (fieldType().indexAnalyzer().name().equals(fieldType().getTextSearchInfo().getSearchAnalyzer().name()) == false) {
builder.field(Fields.SEARCH_ANALYZER.getPreferredName(), fieldType().getTextSearchInfo().getSearchAnalyzer().name());
}
builder.field(Fields.PRESERVE_SEPARATORS.getPreferredName(), fieldType().preserveSep());
builder.field(Fields.PRESERVE_POSITION_INCREMENTS.getPreferredName(), fieldType().preservePositionIncrements());

2
server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
View File

@ -39,7 +39,6 @@ import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.common.time.DateMathParser;
@ -312,7 +311,6 @@ public final class DateFieldMapper extends FieldMapper {
this.dateTimeFormatter = dateTimeFormatter;
this.dateMathParser = dateTimeFormatter.toDateMathParser();
this.resolution = resolution;
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); // allows match queries on date fields
}
public DateFieldType(String name) {

8
server/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
View File

@ -44,19 +44,13 @@ public final class DocumentFieldMappers implements Iterable<Mapper> {
public DocumentFieldMappers(Collection<FieldMapper> mappers,
Collection<FieldAliasMapper> aliasMappers,
Analyzer defaultIndex,
Analyzer defaultSearch,
Analyzer defaultSearchQuote) {
Analyzer defaultIndex) {
Map<String, Mapper> fieldMappers = new HashMap<>();
Map<String, Analyzer> indexAnalyzers = new HashMap<>();
Map<String, Analyzer> searchAnalyzers = new HashMap<>();
Map<String, Analyzer> searchQuoteAnalyzers = new HashMap<>();
for (FieldMapper mapper : mappers) {
fieldMappers.put(mapper.name(), mapper);
MappedFieldType fieldType = mapper.fieldType();
put(indexAnalyzers, fieldType.name(), fieldType.indexAnalyzer(), defaultIndex);
put(searchAnalyzers, fieldType.name(), fieldType.searchAnalyzer(), defaultSearch);
put(searchQuoteAnalyzers, fieldType.name(), fieldType.searchQuoteAnalyzer(), defaultSearchQuote);
}
for (FieldAliasMapper aliasMapper : aliasMappers) {

4
server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View File

@ -159,9 +159,7 @@ public class DocumentMapper implements ToXContentFragment {
final IndexAnalyzers indexAnalyzers = mapperService.getIndexAnalyzers();
this.fieldMappers = new DocumentFieldMappers(newFieldMappers,
newFieldAliasMappers,
indexAnalyzers.getDefaultIndexAnalyzer(),
indexAnalyzers.getDefaultSearchAnalyzer(),
indexAnalyzers.getDefaultSearchQuoteAnalyzer());
indexAnalyzers.getDefaultIndexAnalyzer());
Map<String, ObjectMapper> builder = new HashMap<>();
for (ObjectMapper objectMapper : newObjectMappers) {

10
server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View File

@ -429,15 +429,19 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
}
} else {
boolean hasDefaultIndexAnalyzer = fieldType().indexAnalyzer().name().equals("default");
final String searchAnalyzerName = fieldType().searchAnalyzer().name();
final String searchAnalyzerName = fieldType().getTextSearchInfo().getSearchAnalyzer() == null
? "default" : fieldType().getTextSearchInfo().getSearchAnalyzer().name();
final String searchQuoteAnalyzerName = fieldType().getTextSearchInfo().getSearchQuoteAnalyzer() == null
? searchAnalyzerName : fieldType().getTextSearchInfo().getSearchQuoteAnalyzer().name();
boolean hasDifferentSearchAnalyzer = searchAnalyzerName.equals(fieldType().indexAnalyzer().name()) == false;
boolean hasDifferentSearchQuoteAnalyzer = searchAnalyzerName.equals(fieldType().searchQuoteAnalyzer().name()) == false;
boolean hasDifferentSearchQuoteAnalyzer
= Objects.equals(searchAnalyzerName, searchQuoteAnalyzerName) == false;
if (includeDefaults || hasDefaultIndexAnalyzer == false || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) {
builder.field("analyzer", fieldType().indexAnalyzer().name());
if (includeDefaults || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) {
builder.field("search_analyzer", searchAnalyzerName);
if (includeDefaults || hasDifferentSearchQuoteAnalyzer) {
builder.field("search_quote_analyzer", fieldType().searchQuoteAnalyzer().name());
builder.field("search_quote_analyzer", searchQuoteAnalyzerName);
}
}
}

1
server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
View File

@ -119,7 +119,6 @@ public class IdFieldMapper extends MetadataFieldMapper {
private IdFieldType() {
super(NAME, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
protected IdFieldType(IdFieldType ref) {

28
server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
View File

@ -235,18 +235,16 @@ public final class KeywordFieldMapper extends FieldMapper {
boolean eagerGlobalOrdinals, NamedAnalyzer normalizer, NamedAnalyzer searchAnalyzer,
SimilarityProvider similarity, Map<String, String> meta, float boost) {
super(name, fieldType.indexOptions() != IndexOptions.NONE,
hasDocValues, new TextSearchInfo(fieldType, similarity), meta);
hasDocValues, new TextSearchInfo(fieldType, similarity, searchAnalyzer, searchAnalyzer), meta);
this.hasNorms = fieldType.omitNorms() == false;
setEagerGlobalOrdinals(eagerGlobalOrdinals);
setIndexAnalyzer(normalizer);
setSearchAnalyzer(searchAnalyzer);
setBoost(boost);
}
public KeywordFieldType(String name, boolean isSearchable, boolean hasDocValues, Map<String, String> meta) {
super(name, isSearchable, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
public KeywordFieldType(String name) {
@ -254,12 +252,15 @@ public final class KeywordFieldMapper extends FieldMapper {
Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, null, Collections.emptyMap(), 1f);
}
public KeywordFieldType(String name, NamedAnalyzer analyzer) {
super(name, true, true, new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap());
}
protected KeywordFieldType(KeywordFieldType ref) {
super(ref);
this.hasNorms = ref.hasNorms;
setEagerGlobalOrdinals(ref.eagerGlobalOrdinals());
setIndexAnalyzer(ref.indexAnalyzer());
setSearchAnalyzer(ref.searchAnalyzer());
}
@Override
@ -267,20 +268,6 @@ public final class KeywordFieldMapper extends FieldMapper {
return new KeywordFieldType(this);
}
@Override
public boolean equals(Object o) {
if (super.equals(o) == false) {
return false;
}
KeywordFieldType other = (KeywordFieldType) o;
return Objects.equals(searchAnalyzer(), other.searchAnalyzer());
}
@Override
public int hashCode() {
return 31 * super.hashCode() + Objects.hash(searchAnalyzer());
}
@Override
public String typeName() {
return CONTENT_TYPE;
@ -319,7 +306,7 @@ public final class KeywordFieldMapper extends FieldMapper {
@Override
protected BytesRef indexedValueForSearch(Object value) {
if (searchAnalyzer() == Lucene.KEYWORD_ANALYZER) {
if (getTextSearchInfo().getSearchAnalyzer() == Lucene.KEYWORD_ANALYZER) {
// keyword analyzer with the default attribute source which encodes terms using UTF8
// in that case we skip normalization, which may be slow if there many terms need to
// parse (eg. large terms query) since Analyzer.normalize involves things like creating
@ -334,7 +321,7 @@ public final class KeywordFieldMapper extends FieldMapper {
if (value instanceof BytesRef) {
value = ((BytesRef) value).utf8ToString();
}
return searchAnalyzer().normalize(name(), value.toString());
return getTextSearchInfo().getSearchAnalyzer().normalize(name(), value.toString());
}
}
@ -439,7 +426,6 @@ public final class KeywordFieldMapper extends FieldMapper {
}
this.ignoreAbove = k.ignoreAbove;
this.splitQueriesOnWhitespace = k.splitQueriesOnWhitespace;
this.fieldType().setSearchAnalyzer(k.fieldType().searchAnalyzer());
this.fieldType().setBoost(k.fieldType().boost());
}

24
server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
View File

@ -65,8 +65,6 @@ public abstract class MappedFieldType {
private final TextSearchInfo textSearchInfo;
private float boost;
private NamedAnalyzer indexAnalyzer;
private NamedAnalyzer searchAnalyzer;
private NamedAnalyzer searchQuoteAnalyzer;
private boolean eagerGlobalOrdinals;
private Map<String, String> meta;
@ -76,8 +74,6 @@ public abstract class MappedFieldType {
this.isIndexed = ref.isIndexed;
this.docValues = ref.hasDocValues();
this.indexAnalyzer = ref.indexAnalyzer();
this.searchAnalyzer = ref.searchAnalyzer();
this.searchQuoteAnalyzer = ref.searchQuoteAnalyzer;
this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals;
this.meta = ref.meta;
this.textSearchInfo = ref.textSearchInfo;
@ -119,15 +115,13 @@ public abstract class MappedFieldType {
docValues == fieldType.docValues &&
Objects.equals(name, fieldType.name) &&
Objects.equals(indexAnalyzer, fieldType.indexAnalyzer) &&
Objects.equals(searchAnalyzer, fieldType.searchAnalyzer) &&
Objects.equals(searchQuoteAnalyzer(), fieldType.searchQuoteAnalyzer()) &&
Objects.equals(eagerGlobalOrdinals, fieldType.eagerGlobalOrdinals) &&
Objects.equals(meta, fieldType.meta);
}
@Override
public int hashCode() {
return Objects.hash(name, boost, docValues, indexAnalyzer, searchAnalyzer, searchQuoteAnalyzer,
return Objects.hash(name, boost, docValues, indexAnalyzer,
eagerGlobalOrdinals, meta);
}
@ -165,22 +159,6 @@ public abstract class MappedFieldType {
this.indexAnalyzer = analyzer;
}
public NamedAnalyzer searchAnalyzer() {
return searchAnalyzer;
}
public void setSearchAnalyzer(NamedAnalyzer analyzer) {
this.searchAnalyzer = analyzer;
}
public NamedAnalyzer searchQuoteAnalyzer() {
return searchQuoteAnalyzer == null ? searchAnalyzer : searchQuoteAnalyzer;
}
public void setSearchQuoteAnalyzer(NamedAnalyzer analyzer) {
this.searchQuoteAnalyzer = analyzer;
}
/** Given a value that comes from the stored fields API, convert it to the
* expected type. For instance a date field would store dates as longs and
* format it back to a string in this method. */

10
server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -166,10 +166,12 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
this.indexAnalyzers = indexAnalyzers;
this.fieldTypes = new FieldTypeLookup();
this.documentParser = new DocumentMapperParser(indexSettings, this, xContentRegistry, similarityService, mapperRegistry,
queryShardContextSupplier);
this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), p -> p.indexAnalyzer());
this.searchAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer());
this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer());
queryShardContextSupplier);
this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), MappedFieldType::indexAnalyzer);
this.searchAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchAnalyzer(),
p -> p.getTextSearchInfo().getSearchAnalyzer());
this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchQuoteAnalyzer(),
p -> p.getTextSearchInfo().getSearchQuoteAnalyzer());
this.mapperRegistry = mapperRegistry;
this.idFieldDataEnabled = idFieldDataEnabled;

1
server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
View File

@ -907,7 +907,6 @@ public class NumberFieldMapper extends FieldMapper {
super(name, isSearchable, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
this.type = Objects.requireNonNull(type);
this.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); // allows number fields in significant text aggs - do we need this?
this.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); // allows match queries on number fields
}
public NumberFieldType(String name, NumberType type) {

2
server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
View File

@ -199,7 +199,6 @@ public class RangeFieldMapper extends FieldMapper {
dateTimeFormatter = null;
dateMathParser = null;
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
public RangeFieldType(String name, RangeType type) {
@ -212,7 +211,6 @@ public class RangeFieldMapper extends FieldMapper {
this.dateTimeFormatter = Objects.requireNonNull(formatter);
this.dateMathParser = dateTimeFormatter.toDateMathParser();
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
public RangeFieldType(String name, DateFormatter formatter) {

1
server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
View File

@ -112,7 +112,6 @@ public class RoutingFieldMapper extends MetadataFieldMapper {
private RoutingFieldType() {
super(NAME, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
protected RoutingFieldType(RoutingFieldType ref) {

4
server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
View File

@ -125,8 +125,8 @@ public abstract class StringFieldType extends TermBasedFieldType {
}
Term term;
if (searchAnalyzer() != null) {
value = normalizeWildcardPattern(name(), value, searchAnalyzer());
if (getTextSearchInfo().getSearchAnalyzer() != null) {
value = normalizeWildcardPattern(name(), value, getTextSearchInfo().getSearchAnalyzer());
term = new Term(name(), value);
} else {
term = new Term(name(), indexedValueForSearch(value));

26
server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
View File

@ -59,6 +59,7 @@ import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.elasticsearch.Version;
import org.elasticsearch.common.collect.Iterators;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
@ -129,7 +130,7 @@ public class TextFieldMapper extends FieldMapper {
private double fielddataMinFreq = Defaults.FIELDDATA_MIN_FREQUENCY;
private double fielddataMaxFreq = Defaults.FIELDDATA_MAX_FREQUENCY;
private int fielddataMinSegSize = Defaults.FIELDDATA_MIN_SEGMENT_SIZE;
private SimilarityProvider similarity;
protected SimilarityProvider similarity;
public Builder(String name) {
super(name, Defaults.FIELD_TYPE);
@ -195,10 +196,9 @@ public class TextFieldMapper extends FieldMapper {
}
private TextFieldType buildFieldType(BuilderContext context) {
TextFieldType ft = new TextFieldType(buildFullName(context), fieldType, similarity, meta);
TextFieldType ft
= new TextFieldType(buildFullName(context), fieldType, similarity, searchAnalyzer, searchQuoteAnalyzer, meta);
ft.setIndexAnalyzer(indexAnalyzer);
ft.setSearchAnalyzer(searchAnalyzer);
ft.setSearchQuoteAnalyzer(searchQuoteAnalyzer);
ft.setEagerGlobalOrdinals(eagerGlobalOrdinals);
if (fielddata) {
ft.setFielddata(true);
@ -258,7 +258,7 @@ public class TextFieldMapper extends FieldMapper {
}
@Override
public TextFieldMapper build(BuilderContext context) {
public FieldMapper build(BuilderContext context) {
if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
throw new IllegalArgumentException("Cannot set position_increment_gap on field ["
@ -570,9 +570,10 @@ public class TextFieldMapper extends FieldMapper {
private boolean indexPhrases = false;
private final FieldType indexedFieldType;
public TextFieldType(String name, FieldType indexedFieldType, SimilarityProvider similarity, Map<String, String> meta) {
public TextFieldType(String name, FieldType indexedFieldType, SimilarityProvider similarity, NamedAnalyzer searchAnalyzer,
NamedAnalyzer searchQuoteAnalyzer, Map<String, String> meta) {
super(name, indexedFieldType.indexOptions() != IndexOptions.NONE, false,
new TextSearchInfo(indexedFieldType, similarity), meta);
new TextSearchInfo(indexedFieldType, similarity, searchAnalyzer, searchQuoteAnalyzer), meta);
this.indexedFieldType = indexedFieldType;
fielddata = false;
fielddataMinFrequency = Defaults.FIELDDATA_MIN_FREQUENCY;
@ -581,13 +582,14 @@ public class TextFieldMapper extends FieldMapper {
}
public TextFieldType(String name, boolean indexed, Map<String, String> meta) {
super(name, indexed, false, new TextSearchInfo(Defaults.FIELD_TYPE, null), meta);
super(name, indexed, false,
new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), meta);
this.indexedFieldType = Defaults.FIELD_TYPE;
fielddata = false;
}
public TextFieldType(String name) {
this(name, Defaults.FIELD_TYPE, null, Collections.emptyMap());
this(name, Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER, Collections.emptyMap());
}
protected TextFieldType(TextFieldType ref) {
@ -723,7 +725,7 @@ public class TextFieldMapper extends FieldMapper {
throw new IllegalArgumentException("Cannot create intervals over field [" + name() + "] with no positions indexed");
}
if (analyzer == null) {
analyzer = searchAnalyzer();
analyzer = getTextSearchInfo().getSearchAnalyzer();
}
if (prefix) {
BytesRef normalizedTerm = analyzer.normalize(name(), text);
@ -732,7 +734,7 @@ public class TextFieldMapper extends FieldMapper {
}
return Intervals.prefix(normalizedTerm);
}
IntervalBuilder builder = new IntervalBuilder(name(), analyzer == null ? searchAnalyzer() : analyzer);
IntervalBuilder builder = new IntervalBuilder(name(), analyzer == null ? getTextSearchInfo().getSearchAnalyzer() : analyzer);
return builder.analyzeText(text, maxGaps, ordered);
}
@ -920,8 +922,6 @@ public class TextFieldMapper extends FieldMapper {
if (this.phraseFieldMapper != null && mw.phraseFieldMapper != null) {
this.phraseFieldMapper = (PhraseFieldMapper) this.phraseFieldMapper.merge(mw.phraseFieldMapper);
}
this.fieldType().setSearchAnalyzer(mw.fieldType().searchAnalyzer());
this.fieldType().setSearchQuoteAnalyzer(mw.fieldType().searchQuoteAnalyzer());
}
@Override

32
server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java
View File

@ -21,6 +21,8 @@ package org.elasticsearch.index.mapper;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.similarity.SimilarityProvider;
/**
@ -40,15 +42,28 @@ public class TextSearchInfo {
*
* Note that the results of {@link #isStored()} for this may not be accurate
*/
public static final TextSearchInfo SIMPLE_MATCH_ONLY = new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE, null);
public static final TextSearchInfo SIMPLE_MATCH_ONLY
= new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER);
/**
* Defines indexing information for fields that index as keywords, but split query input
* on whitespace to build disjunctions.
*
* Note that the results of {@link #isStored()} for this may not be accurate
*/
public static final TextSearchInfo WHITESPACE_MATCH_ONLY
= new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE, null, Lucene.WHITESPACE_ANALYZER, Lucene.WHITESPACE_ANALYZER);
/**
* Specifies that this field does not support text searching of any kind
*/
public static final TextSearchInfo NONE = new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE, null);
public static final TextSearchInfo NONE
= new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE, null, null, null);
private final FieldType luceneFieldType;
private final SimilarityProvider similarity;
private final NamedAnalyzer searchAnalyzer;
private final NamedAnalyzer searchQuoteAnalyzer;
/**
* Create a new TextSearchInfo
@ -57,15 +72,26 @@ public class TextSearchInfo {
* @param similarity defines which Similarity to use when searching. If set to {@code null}
* then the default Similarity will be used.
*/
public TextSearchInfo(FieldType luceneFieldType, SimilarityProvider similarity) {
public TextSearchInfo(FieldType luceneFieldType, SimilarityProvider similarity,
NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer) {
this.luceneFieldType = luceneFieldType;
this.similarity = similarity;
this.searchAnalyzer = searchAnalyzer;
this.searchQuoteAnalyzer = searchQuoteAnalyzer;
}
public SimilarityProvider getSimilarity() {
return similarity;
}
public NamedAnalyzer getSearchAnalyzer() {
return searchAnalyzer;
}
public NamedAnalyzer getSearchQuoteAnalyzer() {
return searchQuoteAnalyzer;
}
/**
* @return whether or not this field supports positional queries
*/

8
server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
View File

@ -639,7 +639,7 @@ public abstract class IntervalsSourceProvider implements NamedWriteable, ToXCont
@Override
public IntervalsSource getSource(QueryShardContext context, MappedFieldType fieldType) {
NamedAnalyzer analyzer = fieldType.searchAnalyzer();
NamedAnalyzer analyzer = fieldType.getTextSearchInfo().getSearchAnalyzer();
if (this.analyzer != null) {
analyzer = context.getMapperService().getIndexAnalyzers().get(this.analyzer);
}
@ -649,7 +649,7 @@ public abstract class IntervalsSourceProvider implements NamedWriteable, ToXCont
assert fieldType != null;
checkPositions(fieldType);
if (this.analyzer == null) {
analyzer = fieldType.searchAnalyzer();
analyzer = fieldType.getTextSearchInfo().getSearchAnalyzer();
}
BytesRef normalizedTerm = analyzer.normalize(useField, pattern);
source = Intervals.fixField(useField, Intervals.wildcard(normalizedTerm));
@ -776,7 +776,7 @@ public abstract class IntervalsSourceProvider implements NamedWriteable, ToXCont
@Override
public IntervalsSource getSource(QueryShardContext context, MappedFieldType fieldType) {
NamedAnalyzer analyzer = fieldType.searchAnalyzer();
NamedAnalyzer analyzer = fieldType.getTextSearchInfo().getSearchAnalyzer();
if (this.analyzer != null) {
analyzer = context.getMapperService().getIndexAnalyzers().get(this.analyzer);
}
@ -786,7 +786,7 @@ public abstract class IntervalsSourceProvider implements NamedWriteable, ToXCont
assert fieldType != null;
checkPositions(fieldType);
if (this.analyzer == null) {
analyzer = fieldType.searchAnalyzer();
analyzer = fieldType.getTextSearchInfo().getSearchAnalyzer();
}
}
checkPositions(fieldType);

8
server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
View File

@ -271,8 +271,8 @@ public class QueryShardContext extends QueryRewriteContext {
* TODO: remove this by moving defaults into mappers themselves
*/
public Analyzer getSearchAnalyzer(MappedFieldType fieldType) {
if (fieldType.searchAnalyzer() != null) {
return fieldType.searchAnalyzer();
if (fieldType.getTextSearchInfo().getSearchAnalyzer() != null) {
return fieldType.getTextSearchInfo().getSearchAnalyzer();
}
return getMapperService().searchAnalyzer();
}
@ -282,8 +282,8 @@ public class QueryShardContext extends QueryRewriteContext {
* TODO: remove this by moving defaults into mappers themselves
*/
public Analyzer getSearchQuoteAnalyzer(MappedFieldType fieldType) {
if (fieldType.searchQuoteAnalyzer() != null) {
return fieldType.searchQuoteAnalyzer();
if (fieldType.getTextSearchInfo().getSearchQuoteAnalyzer() != null) {
return fieldType.getTextSearchInfo().getSearchQuoteAnalyzer();
}
return getMapperService().searchQuoteAnalyzer();
}

2
server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java
View File

@ -85,7 +85,7 @@ public class SimpleQueryStringQueryParser extends SimpleQueryParser {
if (getAnalyzer() != null) {
return analyzer;
}
return ft.searchAnalyzer();
return ft.getTextSearchInfo().getSearchAnalyzer();
}
/**

4
server/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
View File

@ -307,10 +307,10 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> implemen
throw new IllegalArgumentException("no mapping found for field [" + field + "]");
} else if (analyzer == null) {
// no analyzer name passed in, so try the field's analyzer, or the default analyzer
if (fieldType.searchAnalyzer() == null) {
if (fieldType.getTextSearchInfo().getSearchAnalyzer() == null) {
suggestionContext.setAnalyzer(mapperService.searchAnalyzer());
} else {
suggestionContext.setAnalyzer(fieldType.searchAnalyzer());
suggestionContext.setAnalyzer(fieldType.getTextSearchInfo().getSearchAnalyzer());
}
} else {
Analyzer luceneAnalyzer = mapperService.getNamedAnalyzer(analyzer);

4
server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java
View File

@ -95,8 +95,8 @@ public class PreBuiltAnalyzerTests extends ESSingleNodeTestCase {
MapperService mapperService = createIndex("test", indexSettings, "type", mapping).mapperService();
MappedFieldType fieldType = mapperService.fieldType("field");
assertThat(fieldType.searchAnalyzer(), instanceOf(NamedAnalyzer.class));
NamedAnalyzer fieldMapperNamedAnalyzer = fieldType.searchAnalyzer();
assertThat(fieldType.getTextSearchInfo().getSearchAnalyzer(), instanceOf(NamedAnalyzer.class));
NamedAnalyzer fieldMapperNamedAnalyzer = fieldType.getTextSearchInfo().getSearchAnalyzer();
assertThat(fieldMapperNamedAnalyzer.analyzer(), is(namedAnalyzer.analyzer()));
}

4
server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
View File

@ -115,7 +115,7 @@ public class CompletionFieldMapperTests extends FieldMapperTestCase<CompletionFi
assertThat(analyzer.preservePositionIncrements(), equalTo(true));
assertThat(analyzer.preserveSep(), equalTo(true));
NamedAnalyzer searchAnalyzer = completionFieldType.searchAnalyzer();
NamedAnalyzer searchAnalyzer = completionFieldType.getTextSearchInfo().getSearchAnalyzer();
assertThat(searchAnalyzer.name(), equalTo("simple"));
assertThat(searchAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class));
analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer();
@ -148,7 +148,7 @@ public class CompletionFieldMapperTests extends FieldMapperTestCase<CompletionFi
assertThat(analyzer.preservePositionIncrements(), equalTo(true));
assertThat(analyzer.preserveSep(), equalTo(false));
NamedAnalyzer searchAnalyzer = completionFieldType.searchAnalyzer();
NamedAnalyzer searchAnalyzer = completionFieldType.getTextSearchInfo().getSearchAnalyzer();
assertThat(searchAnalyzer.name(), equalTo("standard"));
assertThat(searchAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class));
analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer();

8
server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
View File

@ -126,23 +126,17 @@ public class DocumentFieldMapperTests extends LuceneTestCase {
public void testAnalyzers() throws IOException {
FakeFieldType fieldType1 = new FakeFieldType("field1");
fieldType1.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new FakeAnalyzer("index")));
fieldType1.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new FakeAnalyzer("search")));
fieldType1.setSearchQuoteAnalyzer(new NamedAnalyzer("baz", AnalyzerScope.INDEX, new FakeAnalyzer("search_quote")));
FieldMapper fieldMapper1 = new FakeFieldMapper(fieldType1);
FakeFieldType fieldType2 = new FakeFieldType("field2");
FieldMapper fieldMapper2 = new FakeFieldMapper(fieldType2);
Analyzer defaultIndex = new FakeAnalyzer("default_index");
Analyzer defaultSearch = new FakeAnalyzer("default_search");
Analyzer defaultSearchQuote = new FakeAnalyzer("default_search_quote");
DocumentFieldMappers documentFieldMappers = new DocumentFieldMappers(
Arrays.asList(fieldMapper1, fieldMapper2),
Collections.emptyList(),
defaultIndex,
defaultSearch,
defaultSearchQuote);
defaultIndex);
assertAnalyzes(documentFieldMappers.indexAnalyzer(), "field1", "index");

8
server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java
View File

@ -119,7 +119,7 @@ public class DocumentMapperTests extends ESSingleNodeTestCase {
.endObject().endObject();
MapperService mapperService = createIndex("test", Settings.EMPTY, "type", mapping1).mapperService();
assertThat(mapperService.fieldType("field").searchAnalyzer().name(), equalTo("whitespace"));
assertThat(mapperService.fieldType("field").getTextSearchInfo().getSearchAnalyzer().name(), equalTo("whitespace"));
String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field")
@ -130,7 +130,7 @@ public class DocumentMapperTests extends ESSingleNodeTestCase {
.endObject().endObject());
mapperService.merge("type", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE);
assertThat(mapperService.fieldType("field").searchAnalyzer().name(), equalTo("keyword"));
assertThat(mapperService.fieldType("field").getTextSearchInfo().getSearchAnalyzer().name(), equalTo("keyword"));
}
public void testChangeSearchAnalyzerToDefault() throws Exception {
@ -143,7 +143,7 @@ public class DocumentMapperTests extends ESSingleNodeTestCase {
.endObject().endObject();
MapperService mapperService = createIndex("test", Settings.EMPTY, "type", mapping1).mapperService();
assertThat(mapperService.fieldType("field").searchAnalyzer().name(), equalTo("whitespace"));
assertThat(mapperService.fieldType("field").getTextSearchInfo().getSearchAnalyzer().name(), equalTo("whitespace"));
String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field")
@ -153,7 +153,7 @@ public class DocumentMapperTests extends ESSingleNodeTestCase {
.endObject().endObject());
mapperService.merge("type", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE);
assertThat(mapperService.fieldType("field").searchAnalyzer().name(), equalTo("standard"));
assertThat(mapperService.fieldType("field").getTextSearchInfo().getSearchAnalyzer().name(), equalTo("standard"));
}
public void testConcurrentMergeTest() throws Throwable {

8
server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java
View File

@ -64,7 +64,8 @@ public class FakeStringFieldMapper extends FieldMapper {
@Override
public FakeStringFieldMapper build(BuilderContext context) {
return new FakeStringFieldMapper(
fieldType, new FakeStringFieldType(name),
fieldType,
new FakeStringFieldType(name, new TextSearchInfo(fieldType, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER)),
multiFieldsBuilder.build(this, context), copyTo);
}
}
@ -85,10 +86,9 @@ public class FakeStringFieldMapper extends FieldMapper {
public static final class FakeStringFieldType extends StringFieldType {
public FakeStringFieldType(String name) {
super(name, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
public FakeStringFieldType(String name, TextSearchInfo textSearchInfo) {
super(name, true, true, textSearchInfo, Collections.emptyMap());
setIndexAnalyzer(Lucene.STANDARD_ANALYZER);
setSearchAnalyzer(Lucene.STANDARD_ANALYZER);
}
protected FakeStringFieldType(FakeStringFieldType ref) {

17
server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockLowerCaseFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.index.DocValuesType;
@ -558,13 +559,15 @@ public class KeywordFieldMapperTests extends FieldMapperTestCase<KeywordFieldMap
MappedFieldType fieldType = indexService.mapperService().fieldType("field");
assertThat(fieldType, instanceOf(KeywordFieldMapper.KeywordFieldType.class));
KeywordFieldMapper.KeywordFieldType ft = (KeywordFieldMapper.KeywordFieldType) fieldType;
assertTokenStreamContents(ft.searchAnalyzer().analyzer().tokenStream("", "Hello World"), new String[] {"Hello World"});
Analyzer a = ft.getTextSearchInfo().getSearchAnalyzer();
assertTokenStreamContents(a.tokenStream("", "Hello World"), new String[] {"Hello World"});
fieldType = indexService.mapperService().fieldType("field_with_normalizer");
assertThat(fieldType, instanceOf(KeywordFieldMapper.KeywordFieldType.class));
ft = (KeywordFieldMapper.KeywordFieldType) fieldType;
assertThat(ft.searchAnalyzer().name(), equalTo("my_lowercase"));
assertTokenStreamContents(ft.searchAnalyzer().analyzer().tokenStream("", "Hello World"), new String[] {"hello", "world"});
assertThat(ft.getTextSearchInfo().getSearchAnalyzer().name(), equalTo("my_lowercase"));
assertTokenStreamContents(ft.getTextSearchInfo().getSearchAnalyzer().analyzer().tokenStream("", "Hello World"),
new String[] {"hello", "world"});
mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type")
@ -585,13 +588,15 @@ public class KeywordFieldMapperTests extends FieldMapperTestCase<KeywordFieldMap
fieldType = indexService.mapperService().fieldType("field");
assertThat(fieldType, instanceOf(KeywordFieldMapper.KeywordFieldType.class));
ft = (KeywordFieldMapper.KeywordFieldType) fieldType;
assertTokenStreamContents(ft.searchAnalyzer().analyzer().tokenStream("", "Hello World"), new String[] {"Hello", "World"});
assertTokenStreamContents(ft.getTextSearchInfo().getSearchAnalyzer().analyzer().tokenStream("", "Hello World"),
new String[] {"Hello", "World"});
fieldType = indexService.mapperService().fieldType("field_with_normalizer");
assertThat(fieldType, instanceOf(KeywordFieldMapper.KeywordFieldType.class));
ft = (KeywordFieldMapper.KeywordFieldType) fieldType;
assertThat(ft.searchAnalyzer().name(), equalTo("my_lowercase"));
assertTokenStreamContents(ft.searchAnalyzer().analyzer().tokenStream("", "Hello World"), new String[] {"hello world"});
assertThat(ft.getTextSearchInfo().getSearchAnalyzer().name(), equalTo("my_lowercase"));
assertTokenStreamContents(ft.getTextSearchInfo().getSearchAnalyzer().analyzer().tokenStream("", "Hello World"),
new String[] {"hello world"});
}
public void testMeta() throws Exception {

31
server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
View File

@ -24,8 +24,6 @@ import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
@ -44,7 +42,6 @@ import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
import org.elasticsearch.index.mapper.MappedFieldType.Relation;
import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
@ -55,28 +52,6 @@ import java.util.Map;
public class KeywordFieldTypeTests extends FieldTypeTestCase<KeywordFieldType> {
@Before
public void addModifiers() {
addModifier(t -> {
KeywordFieldType copy = t.clone();
if (copy.normalizer() == null) {
copy.setSearchAnalyzer(new NamedAnalyzer("keyword", AnalyzerScope.INDEX, new KeywordAnalyzer()));
} else {
copy.setSearchAnalyzer(null);
}
return copy;
});
addModifier(t -> {
KeywordFieldType copy = t.clone();
if (copy.searchAnalyzer() == Lucene.KEYWORD_ANALYZER) {
copy.setSearchAnalyzer(new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer()));
} else {
copy.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
return copy;
});
}
@Override
protected KeywordFieldType createDefaultFieldType(String name, Map<String, String> meta) {
return new KeywordFieldMapper.KeywordFieldType(name, true, true, meta);
@ -102,7 +77,6 @@ public class KeywordFieldTypeTests extends FieldTypeTestCase<KeywordFieldType> {
}
public void testTermQueryWithNormalizer() {
MappedFieldType ft = new KeywordFieldType("field");
Analyzer normalizer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
@ -115,7 +89,7 @@ public class KeywordFieldTypeTests extends FieldTypeTestCase<KeywordFieldType> {
return new LowerCaseFilter(in);
}
};
ft.setSearchAnalyzer(new NamedAnalyzer("my_normalizer", AnalyzerScope.INDEX, normalizer));
MappedFieldType ft = new KeywordFieldType("field", new NamedAnalyzer("my_normalizer", AnalyzerScope.INDEX, normalizer));
assertEquals(new TermQuery(new Term("field", "foo bar")), ft.termQuery("fOo BaR", null));
}
@ -193,9 +167,8 @@ public class KeywordFieldTypeTests extends FieldTypeTestCase<KeywordFieldType> {
public void testNormalizeQueries() {
MappedFieldType ft = new KeywordFieldType("field");
ft.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
assertEquals(new TermQuery(new Term("field", new BytesRef("FOO"))), ft.termQuery("FOO", null));
ft.setSearchAnalyzer(Lucene.STANDARD_ANALYZER);
ft = new KeywordFieldType("field", Lucene.STANDARD_ANALYZER);
assertEquals(new TermQuery(new Term("field", new BytesRef("foo"))), ft.termQuery("FOO", null));
}
}

6
server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
View File

@ -470,8 +470,10 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
assertSame(current.getDefaultSearchQuoteAnalyzer(), updatedAnalyzers.getDefaultSearchQuoteAnalyzer());
assertFalse(assertSameContainedFilters(originalTokenFilters, current.get("reloadableAnalyzer")));
assertFalse(assertSameContainedFilters(originalTokenFilters, mapperService.fieldType("field").searchAnalyzer()));
assertFalse(assertSameContainedFilters(originalTokenFilters, mapperService.fieldType("otherField").searchQuoteAnalyzer()));
assertFalse(assertSameContainedFilters(originalTokenFilters,
mapperService.fieldType("field").getTextSearchInfo().getSearchAnalyzer()));
assertFalse(assertSameContainedFilters(originalTokenFilters,
mapperService.fieldType("otherField").getTextSearchInfo().getSearchQuoteAnalyzer()));
}
private boolean assertSameContainedFilters(TokenFilterFactory[] originalTokenFilter, NamedAnalyzer updatedAnalyzer) {

24
server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
View File

@ -38,9 +38,10 @@ import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.TextFieldMapper;
import org.elasticsearch.index.mapper.TextSearchInfo;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.mock.orig.Mockito;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.SearchModule;
@ -161,19 +162,10 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
indexSettings);
MapperService mapperService = mock(MapperService.class);
ScriptService scriptService = mock(ScriptService.class);
MappedFieldType fieldType = mockFieldType(suggestionBuilder.field());
boolean fieldTypeSearchAnalyzerSet = randomBoolean();
if (fieldTypeSearchAnalyzerSet) {
NamedAnalyzer searchAnalyzer = new NamedAnalyzer("fieldSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer());
if (Mockito.mockingDetails(fieldType).isMock()) {
when(fieldType.searchAnalyzer()).thenReturn(searchAnalyzer);
} else {
fieldType.setSearchAnalyzer(searchAnalyzer);
}
} else {
when(mapperService.searchAnalyzer())
.thenReturn(new NamedAnalyzer("mapperServiceSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer()));
}
MappedFieldType fieldType = mockFieldType(suggestionBuilder.field(), fieldTypeSearchAnalyzerSet);
when(mapperService.searchAnalyzer())
.thenReturn(new NamedAnalyzer("mapperServiceSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer()));
when(mapperService.fieldType(any(String.class))).thenReturn(fieldType);
when(mapperService.getNamedAnalyzer(any(String.class))).then(
invocation -> new NamedAnalyzer((String) invocation.getArguments()[0], AnalyzerScope.INDEX, new SimpleAnalyzer()));
@ -213,9 +205,13 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
*/
protected abstract void assertSuggestionContext(SB builder, SuggestionContext context) throws IOException;
protected MappedFieldType mockFieldType(String fieldName) {
protected MappedFieldType mockFieldType(String fieldName, boolean analyzerSet) {
MappedFieldType fieldType = mock(MappedFieldType.class);
when(fieldType.name()).thenReturn(fieldName);
NamedAnalyzer searchAnalyzer = analyzerSet ?
new NamedAnalyzer("fieldSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer()) : null;
TextSearchInfo tsi = new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, searchAnalyzer, searchAnalyzer);
when(fieldType.getTextSearchInfo()).thenReturn(tsi);
return fieldType;
}

18
server/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
View File

@ -19,9 +19,13 @@
package org.elasticsearch.search.suggest.completion;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.CompletionFieldMapper;
import org.elasticsearch.index.mapper.CompletionFieldMapper.CompletionFieldType;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
@ -163,8 +167,18 @@ public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTe
}
@Override
protected MappedFieldType mockFieldType(String fieldName) {
CompletionFieldType completionFieldType = new CompletionFieldType(fieldName);
protected MappedFieldType mockFieldType(String fieldName, boolean analyzerSet) {
if (analyzerSet == false) {
CompletionFieldType completionFieldType = new CompletionFieldType(fieldName,
CompletionFieldMapper.Defaults.FIELD_TYPE, null, null, Collections.emptyMap());
completionFieldType.setContextMappings(new ContextMappings(contextMappings));
return completionFieldType;
}
CompletionFieldType completionFieldType = new CompletionFieldType(fieldName,
CompletionFieldMapper.Defaults.FIELD_TYPE,
new NamedAnalyzer("fieldSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer()),
new NamedAnalyzer("fieldSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer()),
Collections.emptyMap());
completionFieldType.setContextMappings(new ContextMappings(contextMappings));
return completionFieldType;
}

23
test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
View File

@ -18,9 +18,6 @@
*/
package org.elasticsearch.index.mapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.EqualsHashCodeTestUtils;
@ -52,26 +49,6 @@ public abstract class FieldTypeTestCase<T extends MappedFieldType> extends ESTes
copy.setBoost(t.boost() + 1);
return (T) copy;
},
t -> {
MappedFieldType copy = t.clone();
NamedAnalyzer a = t.searchAnalyzer();
if (a == null) {
copy.setSearchAnalyzer(new NamedAnalyzer("mutated", AnalyzerScope.INDEX, new StandardAnalyzer()));
return (T) copy;
}
copy.setSearchAnalyzer(new NamedAnalyzer(a.name() + "-mutated", a.scope(), a.analyzer()));
return (T) copy;
},
t -> {
MappedFieldType copy = t.clone();
NamedAnalyzer a = t.searchQuoteAnalyzer();
if (a == null) {
copy.setSearchQuoteAnalyzer(new NamedAnalyzer("mutated", AnalyzerScope.INDEX, new StandardAnalyzer()));
return (T) copy;
}
copy.setSearchQuoteAnalyzer(new NamedAnalyzer(a.name() + "-mutated", a.scope(), a.analyzer()));
return (T) copy;
},
t -> {
MappedFieldType copy = t.clone();
copy.setEagerGlobalOrdinals(t.eagerGlobalOrdinals() == false);

20
x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java
View File

@ -6,7 +6,6 @@
package org.elasticsearch.xpack.flattened.mapper;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexOptions;
@ -28,8 +27,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@ -232,13 +229,10 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper {
public KeyedFlatObjectFieldType(String name, boolean indexed, boolean hasDocValues, String key,
boolean splitQueriesOnWhitespace, Map<String, String> meta) {
super(name, indexed, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
super(name, indexed, hasDocValues,
splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY,
meta);
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
if (splitQueriesOnWhitespace == false) {
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
} else {
setSearchAnalyzer(new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer()));
}
this.key = key;
this.splitQueriesOnWhitespace = splitQueriesOnWhitespace;
}
@ -483,14 +477,10 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper {
public RootFlatObjectFieldType(String name, boolean indexed, boolean hasDocValues, Map<String, String> meta,
boolean splitQueriesOnWhitespace) {
super(name, indexed, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
super(name, indexed, hasDocValues,
splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
this.splitQueriesOnWhitespace = splitQueriesOnWhitespace;
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
if (splitQueriesOnWhitespace) {
setSearchAnalyzer(new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer()));
} else {
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
}
private RootFlatObjectFieldType(RootFlatObjectFieldType ref) {

8
x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java
View File

@ -496,13 +496,13 @@ public class FlatObjectFieldMapperTests extends FieldMapperTestCase<FlatObjectFi
mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
RootFlatObjectFieldType rootFieldType = (RootFlatObjectFieldType) mapperService.fieldType("field");
assertThat(rootFieldType.searchAnalyzer().name(), equalTo("whitespace"));
assertTokenStreamContents(rootFieldType.searchAnalyzer().analyzer().tokenStream("", "Hello World"),
assertThat(rootFieldType.getTextSearchInfo().getSearchAnalyzer().name(), equalTo("_whitespace"));
assertTokenStreamContents(rootFieldType.getTextSearchInfo().getSearchAnalyzer().analyzer().tokenStream("", "Hello World"),
new String[] {"Hello", "World"});
KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) mapperService.fieldType("field.key");
assertThat(keyedFieldType.searchAnalyzer().name(), equalTo("whitespace"));
assertTokenStreamContents(keyedFieldType.searchAnalyzer().analyzer().tokenStream("", "Hello World"),
assertThat(keyedFieldType.getTextSearchInfo().getSearchAnalyzer().name(), equalTo("_whitespace"));
assertTokenStreamContents(keyedFieldType.getTextSearchInfo().getSearchAnalyzer().analyzer().tokenStream("", "Hello World"),
new String[] {"Hello", "World"});
}

4
x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
View File

@ -212,9 +212,9 @@ public class WildcardFieldMapper extends FieldMapper {
static Analyzer lowercaseNormalizer = new LowercaseNormalizer();
public WildcardFieldType(String name, FieldType fieldType, Map<String, String> meta) {
super(name, true, true, new TextSearchInfo(fieldType, null), meta);
super(name, true, true,
new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta);
setIndexAnalyzer(WILDCARD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}
protected WildcardFieldType(WildcardFieldType ref) {