honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-25 09:28:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Restore use of default search and search_quote analyzers (#65491) (#65562)

Browse Source 
In the refactoring of TextFieldMapper, we lost the ability to define
a default search or search_quote analyzer in index settings. This
commit restores that ability, and adds some more comprehensive
testing.

Fixes #65434
This commit is contained in:
Alan Woodward 2020-11-26 18:34:59 +00:00 committed by GitHub
parent f6921af885
commit fb84b6710d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 286 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/41_query_string_with_default_analyzer.yml Normal file
View File

@ -0,0 +1,39 @@
---
"Test default search analyzer is applied":
- do:
indices.create:
index: test
body:
settings:
index.analysis.analyzer.default.type: simple
index.analysis.analyzer.default_search.type: german
mappings:
properties:
body:
type: text
- do:
index:
index: test
id: 1
body:
body: Ich lese die Bücher
- do:
indices.refresh:
index: [ test ]
- do:
search:
index: test
q: "body:Bücher"
- match: { hits.total.value: 0 }
- do:
search:
index: test
q: "body:Bücher"
analyzer: simple
- match: { hits.total.value: 1 }

10
modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java
View File

@ -48,6 +48,7 @@ import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.elasticsearch.common.collect.Iterators;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
@ -61,7 +62,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Supplier;
import static org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType.hasGaps;
@ -93,7 +93,7 @@ public class SearchAsYouTypeFieldMapper extends ParametrizedFieldMapper {
}
public static final TypeParser PARSER
= new TypeParser((n, c) -> new Builder(n, () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer()));
= new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers()));
private static SearchAsYouTypeFieldMapper toType(FieldMapper in) {
return (SearchAsYouTypeFieldMapper) in;
@ -138,9 +138,9 @@ public class SearchAsYouTypeFieldMapper extends ParametrizedFieldMapper {
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
public Builder(String name, Supplier<NamedAnalyzer> defaultAnalyzer) {
public Builder(String name, IndexAnalyzers indexAnalyzers) {
super(name);
this.analyzers = new TextParams.Analyzers(defaultAnalyzer);
this.analyzers = new TextParams.Analyzers(indexAnalyzers);
}
@Override
@ -592,7 +592,7 @@ public class SearchAsYouTypeFieldMapper extends ParametrizedFieldMapper {
@Override
public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this);
return new Builder(simpleName(), builder.analyzers.indexAnalyzers).init(this);
}
public static String getShingleFieldName(String parentField, int shingleSize) {

10
plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
View File

@ -33,6 +33,7 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperParsingException;
@ -55,7 +56,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -102,9 +102,9 @@ public class AnnotatedTextFieldMapper extends ParametrizedFieldMapper {
private final Parameter<Float> boost = Parameter.boostParam();
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
public Builder(String name, Supplier<NamedAnalyzer> defaultAnalyzer) {
public Builder(String name, IndexAnalyzers indexAnalyzers) {
super(name);
this.analyzers = new TextParams.Analyzers(defaultAnalyzer);
this.analyzers = new TextParams.Analyzers(indexAnalyzers);
}
@Override
@ -157,7 +157,7 @@ public class AnnotatedTextFieldMapper extends ParametrizedFieldMapper {
}
}
public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer()));
public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers()));
/**
* Parses markdown-like syntax into plain text and AnnotationTokens with offsets for
@ -571,6 +571,6 @@ public class AnnotatedTextFieldMapper extends ParametrizedFieldMapper {
@Override
public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this);
return new Builder(simpleName(), builder.analyzers.indexAnalyzers).init(this);
}
}

10
plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java
View File

@ -22,9 +22,6 @@ package org.elasticsearch.index.mapper.annotatedtext;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.queries.intervals.Intervals;
import org.apache.lucene.queries.intervals.IntervalsSource;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -46,11 +43,8 @@ public class AnnotatedTextFieldTypeTests extends FieldTypeTestCase {
}
public void testFetchSourceValue() throws IOException {
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder("field", () -> Lucene.STANDARD_ANALYZER)
.build(context)
MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder("field", createDefaultIndexAnalyzers())
.build(new Mapper.BuilderContext(Settings.EMPTY, new ContentPath()))
.fieldType();
assertEquals(Collections.singletonList("value"), fetchSourceValue(fieldType, "value"));

3
server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View File

@ -703,8 +703,7 @@ final class DocumentParser {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING);
if (builder == null) {
builder = new TextFieldMapper.Builder(currentFieldName,
() -> context.mapperService().getIndexAnalyzers().getDefaultIndexAnalyzer())
builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers())
.addMultiField(new KeywordFieldMapper.Builder("keyword").ignoreAbove(256));
}
return builder;

13
server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
View File

@ -65,6 +65,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
@ -271,14 +272,14 @@ public class TextFieldMapper extends ParametrizedFieldMapper {
final TextParams.Analyzers analyzers;
public Builder(String name, Supplier<NamedAnalyzer> defaultAnalyzer) {
this(name, Version.CURRENT, defaultAnalyzer);
public Builder(String name, IndexAnalyzers indexAnalyzers) {
this(name, Version.CURRENT, indexAnalyzers);
}
public Builder(String name, Version indexCreatedVersion, Supplier<NamedAnalyzer> defaultAnalyzer) {
public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
super(name);
this.indexCreatedVersion = indexCreatedVersion;
this.analyzers = new TextParams.Analyzers(defaultAnalyzer);
this.analyzers = new TextParams.Analyzers(indexAnalyzers);
}
public Builder index(boolean index) {
@ -400,7 +401,7 @@ public class TextFieldMapper extends ParametrizedFieldMapper {
}
public static final TypeParser PARSER
= new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer()));
= new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers()));
private static class PhraseWrappedAnalyzer extends AnalyzerWrapper {
@ -862,7 +863,7 @@ public class TextFieldMapper extends ParametrizedFieldMapper {
@Override
public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzer::getDefaultValue).init(this);
return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this);
}
@Override

26
server/src/main/java/org/elasticsearch/index/mapper/TextParams.java
View File

@ -22,6 +22,8 @@ package org.elasticsearch.index.mapper;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.index.analysis.AnalysisMode;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter;
import org.elasticsearch.index.similarity.SimilarityProvider;
@ -42,21 +44,37 @@ public final class TextParams {
public final Parameter<NamedAnalyzer> searchAnalyzer;
public final Parameter<NamedAnalyzer> searchQuoteAnalyzer;
public Analyzers(Supplier<NamedAnalyzer> defaultAnalyzer) {
public final IndexAnalyzers indexAnalyzers;
public Analyzers(IndexAnalyzers indexAnalyzers) {
this.indexAnalyzer = Parameter.analyzerParam("analyzer", false,
m -> m.fieldType().indexAnalyzer(), defaultAnalyzer)
m -> m.fieldType().indexAnalyzer(), indexAnalyzers::getDefaultIndexAnalyzer)
.setSerializerCheck((id, ic, a) -> id || ic ||
Objects.equals(a, getSearchAnalyzer()) == false || Objects.equals(a, getSearchQuoteAnalyzer()) == false)
.setValidator(a -> a.checkAllowedInMode(AnalysisMode.INDEX_TIME));
this.searchAnalyzer
= Parameter.analyzerParam("search_analyzer", true,
m -> m.fieldType().getTextSearchInfo().getSearchAnalyzer(), indexAnalyzer::getValue)
m -> m.fieldType().getTextSearchInfo().getSearchAnalyzer(), () -> {
NamedAnalyzer defaultAnalyzer = indexAnalyzers.get(AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME);
if (defaultAnalyzer != null) {
return defaultAnalyzer;
}
return indexAnalyzer.get();
})
.setSerializerCheck((id, ic, a) -> id || ic || Objects.equals(a, getSearchQuoteAnalyzer()) == false)
.setValidator(a -> a.checkAllowedInMode(AnalysisMode.SEARCH_TIME));
this.searchQuoteAnalyzer
= Parameter.analyzerParam("search_quote_analyzer", true,
m -> m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer(), searchAnalyzer::getValue)
m -> m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer(), () -> {
NamedAnalyzer defaultAnalyzer = indexAnalyzers.get(AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME);
if (defaultAnalyzer != null) {
return defaultAnalyzer;
}
return searchAnalyzer.get();
})
.setValidator(a -> a.checkAllowedInMode(AnalysisMode.SEARCH_TIME));
this.indexAnalyzers = indexAnalyzers;
}
public NamedAnalyzer getIndexAnalyzer() {

2
server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
View File

@ -297,7 +297,7 @@ public class QueryShardContext extends QueryRewriteContext {
return fieldMapping;
} else if (mapUnmappedFieldAsString) {
TextFieldMapper.Builder builder
= new TextFieldMapper.Builder(name, () -> mapperService.getIndexAnalyzers().getDefaultIndexAnalyzer());
= new TextFieldMapper.Builder(name, mapperService.getIndexAnalyzers());
return builder.build(new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(1))).fieldType();
} else {
throw new QueryShardException(this, "No field mapping can be found for the field with name [{}]", name);

3
server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
View File

@ -31,7 +31,6 @@ import org.apache.lucene.index.LogByteSizeMergePolicy;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
@ -100,7 +99,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
if (docValues) {
fieldType = new KeywordFieldMapper.Builder(fieldName).build(context).fieldType();
} else {
fieldType = new TextFieldMapper.Builder(fieldName, () -> Lucene.STANDARD_ANALYZER)
fieldType = new TextFieldMapper.Builder(fieldName, createDefaultIndexAnalyzers())
.fielddata(true).build(context).fieldType();
}
} else if (type.equals("float")) {

11
server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
View File

@ -23,7 +23,6 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedSetDocValues;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper.BuilderContext;
@ -65,7 +64,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase {
{
indexService.clearCaches(false, true);
MappedFieldType ft = new TextFieldMapper.Builder("high_freq", () -> Lucene.STANDARD_ANALYZER)
MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers())
.fielddata(true)
.fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0)
.build(builderCtx).fieldType();
@ -80,7 +79,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase {
}
{
indexService.clearCaches(false, true);
MappedFieldType ft = new TextFieldMapper.Builder("high_freq", () -> Lucene.STANDARD_ANALYZER)
MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers())
.fielddata(true)
.fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, 201, 100)
.build(builderCtx).fieldType();
@ -95,7 +94,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase {
{
indexService.clearCaches(false, true);// test # docs with value
MappedFieldType ft = new TextFieldMapper.Builder("med_freq", () -> Lucene.STANDARD_ANALYZER)
MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers())
.fielddata(true)
.fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, Integer.MAX_VALUE, 101)
.build(builderCtx).fieldType();
@ -111,7 +110,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase {
{
indexService.clearCaches(false, true);
MappedFieldType ft = new TextFieldMapper.Builder("med_freq", () -> Lucene.STANDARD_ANALYZER)
MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers())
.fielddata(true)
.fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, Integer.MAX_VALUE, 101)
.build(builderCtx).fieldType();
@ -128,7 +127,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase {
}
@Override
public void testEmpty() throws Exception {
public void testEmpty() {
assumeTrue("No need to test empty usage here", false);
}
}

7
server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
View File

@ -31,7 +31,6 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
@ -143,9 +142,9 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
final MappedFieldType mapper1
= new TextFieldMapper.Builder("field_1", () -> Lucene.STANDARD_ANALYZER).fielddata(true).build(ctx).fieldType();
= new TextFieldMapper.Builder("field_1", createDefaultIndexAnalyzers()).fielddata(true).build(ctx).fieldType();
final MappedFieldType mapper2
= new TextFieldMapper.Builder("field_2", () -> Lucene.STANDARD_ANALYZER).fielddata(true).build(ctx).fieldType();
= new TextFieldMapper.Builder("field_2", createDefaultIndexAnalyzers()).fielddata(true).build(ctx).fieldType();
final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
Document doc = new Document();
doc.add(new StringField("field_1", "thisisastring", Store.NO));
@ -209,7 +208,7 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
final MappedFieldType mapper1
= new TextFieldMapper.Builder("s", () -> Lucene.STANDARD_ANALYZER).fielddata(true).build(ctx).fieldType();
= new TextFieldMapper.Builder("s", createDefaultIndexAnalyzers()).fielddata(true).build(ctx).fieldType();
final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
Document doc = new Document();
doc.add(new StringField("s", "thisisastring", Store.NO));

150
server/src/test/java/org/elasticsearch/index/mapper/DefaultAnalyzersTests.java Normal file
View File

@ -0,0 +1,150 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
public class DefaultAnalyzersTests extends MapperServiceTestCase {
private boolean setDefaultSearchAnalyzer;
private boolean setDefaultSearchQuoteAnalyzer;
@Override
protected IndexAnalyzers createIndexAnalyzers(IndexSettings indexSettings) {
Map<String, NamedAnalyzer> analyzers = new HashMap<>();
analyzers.put(AnalysisRegistry.DEFAULT_ANALYZER_NAME, new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer()));
if (setDefaultSearchAnalyzer) {
analyzers.put(AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME,
new NamedAnalyzer("default_search", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
if (setDefaultSearchQuoteAnalyzer) {
analyzers.put(AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME,
new NamedAnalyzer("default_search_quote", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
analyzers.put("configured", new NamedAnalyzer("configured", AnalyzerScope.INDEX, new StandardAnalyzer()));
return new IndexAnalyzers(
analyzers,
Collections.emptyMap(),
Collections.emptyMap()
);
}
public void testDefaultSearchAnalyzer() throws IOException {
{
setDefaultSearchAnalyzer = false;
MapperService ms = createMapperService(fieldMapping(b -> b.field("type", "text")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("default", ft.getTextSearchInfo().getSearchAnalyzer().name());
}
{
setDefaultSearchAnalyzer = false;
MapperService ms = createMapperService(fieldMapping(b -> b.field("type", "text").field("search_analyzer", "configured")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("configured", ft.getTextSearchInfo().getSearchAnalyzer().name());
}
{
setDefaultSearchAnalyzer = true;
MapperService ms = createMapperService(fieldMapping(b -> b.field("type", "text")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("default_search", ft.getTextSearchInfo().getSearchAnalyzer().name());
}
{
setDefaultSearchAnalyzer = true;
MapperService ms = createMapperService(fieldMapping(b -> b.field("type", "text").field("search_analyzer", "configured")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("configured", ft.getTextSearchInfo().getSearchAnalyzer().name());
}
}
public void testDefaultSearchQuoteAnalyzer() throws IOException {
{
setDefaultSearchQuoteAnalyzer = false;
setDefaultSearchAnalyzer = false;
MapperService ms = createMapperService(fieldMapping(b -> b.field("type", "text")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("default", ft.getTextSearchInfo().getSearchQuoteAnalyzer().name());
}
{
setDefaultSearchQuoteAnalyzer = false;
setDefaultSearchAnalyzer = false;
MapperService ms
= createMapperService(fieldMapping(b -> b.field("type", "text").field("search_quote_analyzer", "configured")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("configured", ft.getTextSearchInfo().getSearchQuoteAnalyzer().name());
}
{
setDefaultSearchQuoteAnalyzer = true;
setDefaultSearchAnalyzer = false;
MapperService ms = createMapperService(fieldMapping(b -> b.field("type", "text")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("default_search_quote", ft.getTextSearchInfo().getSearchQuoteAnalyzer().name());
}
{
setDefaultSearchQuoteAnalyzer = true;
setDefaultSearchAnalyzer = false;
MapperService ms
= createMapperService(fieldMapping(b -> b.field("type", "text").field("search_quote_analyzer", "configured")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("configured", ft.getTextSearchInfo().getSearchQuoteAnalyzer().name());
}
{
setDefaultSearchQuoteAnalyzer = false;
setDefaultSearchAnalyzer = true;
MapperService ms = createMapperService(fieldMapping(b -> b.field("type", "text")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("default_search", ft.getTextSearchInfo().getSearchQuoteAnalyzer().name());
}
{
setDefaultSearchQuoteAnalyzer = false;
setDefaultSearchAnalyzer = true;
MapperService ms
= createMapperService(fieldMapping(b -> b.field("type", "text").field("search_quote_analyzer", "configured")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("configured", ft.getTextSearchInfo().getSearchQuoteAnalyzer().name());
}
{
setDefaultSearchQuoteAnalyzer = true;
setDefaultSearchAnalyzer = true;
MapperService ms = createMapperService(fieldMapping(b -> b.field("type", "text")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("default_search_quote", ft.getTextSearchInfo().getSearchQuoteAnalyzer().name());
}
{
setDefaultSearchQuoteAnalyzer = true;
setDefaultSearchAnalyzer = true;
MapperService ms
= createMapperService(fieldMapping(b -> b.field("type", "text").field("search_quote_analyzer", "configured")));
MappedFieldType ft = ms.fieldType("field");
assertEquals("configured", ft.getTextSearchInfo().getSearchQuoteAnalyzer().name());
}
}
}

13
server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
View File

@ -19,12 +19,15 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.elasticsearch.Version;
import org.elasticsearch.common.collect.Iterators;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.builders.PointBuilder;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.geometry.Point;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.search.lookup.SearchLookup;
import java.io.IOException;
@ -51,6 +54,12 @@ public class ExternalMapper extends ParametrizedFieldMapper {
public static final String FIELD_SHAPE = "shape";
}
private static final IndexAnalyzers INDEX_ANALYZERS = new IndexAnalyzers(
Collections.singletonMap("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())),
Collections.emptyMap(),
Collections.emptyMap()
);
public static class Builder extends ParametrizedFieldMapper.Builder {
private final BinaryFieldMapper.Builder binBuilder = new BinaryFieldMapper.Builder(Names.FIELD_BIN);
@ -64,7 +73,7 @@ public class ExternalMapper extends ParametrizedFieldMapper {
public Builder(String name, String generatedValue, String mapperName) {
super(name);
this.stringBuilder = new TextFieldMapper.Builder(name, () -> Lucene.STANDARD_ANALYZER).store(false);
this.stringBuilder = new TextFieldMapper.Builder(name, INDEX_ANALYZERS).store(false);
this.generatedValue = generatedValue;
this.mapperName = mapperName;
}

11
server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
View File

@ -26,13 +26,11 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
import org.elasticsearch.test.ESSingleNodeTestCase;
@ -42,7 +40,6 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
@ -130,12 +127,10 @@ public class MultiFieldTests extends ESSingleNodeTestCase {
public void testBuildThenParse() throws Exception {
IndexService indexService = createIndex("test");
Supplier<NamedAnalyzer> a = () -> Lucene.STANDARD_ANALYZER;
DocumentMapper builderDocMapper = new DocumentMapper.Builder(new RootObjectMapper.Builder("person").add(
new TextFieldMapper.Builder("name", a).store(true)
.addMultiField(new TextFieldMapper.Builder("indexed", a).index(true))
.addMultiField(new TextFieldMapper.Builder("not_indexed", a).index(false).store(true))
new TextFieldMapper.Builder("name", createDefaultIndexAnalyzers()).store(true)
.addMultiField(new TextFieldMapper.Builder("indexed", createDefaultIndexAnalyzers()).index(true))
.addMultiField(new TextFieldMapper.Builder("not_indexed", createDefaultIndexAnalyzers()).index(false).store(true))
), indexService.mapperService()).build(indexService.mapperService());
String builtMapping = builderDocMapper.mappingSource().string();

21
server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java
View File

@ -20,10 +20,8 @@ package org.elasticsearch.index.mapper;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
import org.junit.AfterClass;
import java.util.Collections;
import java.util.HashMap;
@ -36,19 +34,12 @@ import static org.hamcrest.Matchers.notNullValue;
public class ObjectMapperMergeTests extends ESTestCase {
private static FieldMapper barFieldMapper = createTextFieldMapper("bar");
private static FieldMapper bazFieldMapper = createTextFieldMapper("baz");
private final FieldMapper barFieldMapper = createTextFieldMapper("bar");
private final FieldMapper bazFieldMapper = createTextFieldMapper("baz");
private static RootObjectMapper rootObjectMapper = createMapping(false, true, true, false);
private final RootObjectMapper rootObjectMapper = createMapping(false, true, true, false);
@AfterClass
public static void cleanupReferences() {
barFieldMapper = null;
bazFieldMapper = null;
rootObjectMapper = null;
}
private static RootObjectMapper createMapping(boolean disabledFieldEnabled, boolean fooFieldEnabled,
private RootObjectMapper createMapping(boolean disabledFieldEnabled, boolean fooFieldEnabled,
boolean includeBarField, boolean includeBazField) {
Map<String, Mapper> mappers = new HashMap<>();
mappers.put("disabled", createObjectMapper("disabled", disabledFieldEnabled, emptyMap()));
@ -151,9 +142,9 @@ public class ObjectMapperMergeTests extends ESTestCase {
.build(context);
}
private static TextFieldMapper createTextFieldMapper(String name) {
private TextFieldMapper createTextFieldMapper(String name) {
final Settings indexSettings = Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT).build();
final Mapper.BuilderContext context = new Mapper.BuilderContext(indexSettings, new ContentPath());
return new TextFieldMapper.Builder(name, () -> Lucene.STANDARD_ANALYZER).build(context);
return new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers()).build(context);
}
}

23
server/src/test/java/org/elasticsearch/index/mapper/TextFieldAnalyzerModeTests.java
View File

@ -24,7 +24,6 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
@ -42,8 +41,6 @@ import java.util.HashMap;
import java.util.Map;
import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_ANALYZER_NAME;
import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME;
import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME;
import static org.hamcrest.Matchers.containsString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ -53,8 +50,6 @@ public class TextFieldAnalyzerModeTests extends ESTestCase {
private static Map<String, NamedAnalyzer> defaultAnalyzers() {
Map<String, NamedAnalyzer> analyzers = new HashMap<>();
analyzers.put(DEFAULT_ANALYZER_NAME, new NamedAnalyzer("default", AnalyzerScope.INDEX, null));
analyzers.put(DEFAULT_SEARCH_ANALYZER_NAME, new NamedAnalyzer("default", AnalyzerScope.INDEX, null));
analyzers.put(DEFAULT_SEARCH_QUOTED_ANALYZER_NAME, new NamedAnalyzer("default", AnalyzerScope.INDEX, null));
return analyzers;
}
@ -94,8 +89,7 @@ public class TextFieldAnalyzerModeTests extends ESTestCase {
IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
TextFieldMapper.Builder builder = new TextFieldMapper.Builder("textField", () -> Lucene.STANDARD_ANALYZER);
builder.parse("field", parserContext, fieldNode);
TextFieldMapper.PARSER.parse("field", fieldNode, parserContext);
// check that "analyzer" set to something that only supports AnalysisMode.SEARCH_TIME or AnalysisMode.INDEX_TIME is blocked
AnalysisMode mode = randomFrom(AnalysisMode.SEARCH_TIME, AnalysisMode.INDEX_TIME);
@ -106,8 +100,7 @@ public class TextFieldAnalyzerModeTests extends ESTestCase {
when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
fieldNode.put("analyzer", "my_analyzer");
MapperException ex = expectThrows(MapperException.class, () -> {
TextFieldMapper.Builder bad = new TextFieldMapper.Builder("textField", () -> Lucene.STANDARD_ANALYZER);
bad.parse("name", parserContext, fieldNode);
TextFieldMapper.PARSER.parse("name", fieldNode, parserContext);
});
assertThat(ex.getMessage(),
containsString("analyzer [my_named_analyzer] contains filters [my_analyzer] that are not allowed to run"));
@ -133,8 +126,7 @@ public class TextFieldAnalyzerModeTests extends ESTestCase {
IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
TextFieldMapper.Builder builder = new TextFieldMapper.Builder("textField", () -> Lucene.STANDARD_ANALYZER);
builder.parse("textField", parserContext, fieldNode);
TextFieldMapper.PARSER.parse("textField", fieldNode, parserContext);
// check that "analyzer" set to AnalysisMode.INDEX_TIME is blocked
mode = AnalysisMode.INDEX_TIME;
@ -151,8 +143,7 @@ public class TextFieldAnalyzerModeTests extends ESTestCase {
fieldNode.put("search_analyzer", "standard");
}
MapperException ex = expectThrows(MapperException.class, () -> {
TextFieldMapper.Builder bad = new TextFieldMapper.Builder("field", () -> Lucene.STANDARD_ANALYZER);
bad.parse("field", parserContext, fieldNode);
TextFieldMapper.PARSER.parse("field", fieldNode, parserContext);
});
assertEquals("analyzer [my_named_analyzer] contains filters [my_analyzer] that are not allowed to run in search time mode.",
ex.getMessage());
@ -173,8 +164,7 @@ public class TextFieldAnalyzerModeTests extends ESTestCase {
IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
MapperException ex = expectThrows(MapperException.class, () -> {
TextFieldMapper.Builder builder = new TextFieldMapper.Builder("textField", () -> Lucene.STANDARD_ANALYZER);
builder.parse("field", parserContext, fieldNode);
TextFieldMapper.PARSER.parse("field", fieldNode, parserContext);
});
assertThat(ex.getMessage(),
containsString("analyzer [my_named_analyzer] contains filters [my_analyzer] that are not allowed to run"));
@ -190,8 +180,7 @@ public class TextFieldAnalyzerModeTests extends ESTestCase {
indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
TextFieldMapper.Builder builder = new TextFieldMapper.Builder("textField", () -> Lucene.STANDARD_ANALYZER);
builder.parse("field", parserContext, fieldNode);
TextFieldMapper.PARSER.parse("field", fieldNode, parserContext);
}
}

6
server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper;
import org.elasticsearch.Version;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
@ -55,7 +54,6 @@ public class TypeParsersTests extends ESTestCase {
}
public void testMultiFieldWithinMultiField() throws IOException {
TextFieldMapper.Builder builder = new TextFieldMapper.Builder("textField", () -> Lucene.STANDARD_ANALYZER);
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
.field("type", "keyword")
@ -76,13 +74,13 @@ public class TypeParsersTests extends ESTestCase {
Map<String, Object> fieldNode = XContentHelper.convertToMap(
BytesReference.bytes(mapping), true, mapping.contentType()).v2();
IndexAnalyzers indexAnalyzers = new IndexAnalyzers(defaultAnalyzers(), Collections.emptyMap(), Collections.emptyMap());
MapperService mapperService = mock(MapperService.class);
IndexAnalyzers indexAnalyzers = new IndexAnalyzers(defaultAnalyzers(), Collections.emptyMap(), Collections.emptyMap());
when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers);
Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext(
null, mapperService, type -> typeParser, Version.CURRENT, null, null, null);
builder.parse("some-field", olderContext, fieldNode);
TextFieldMapper.PARSER.parse("some-field", fieldNode, olderContext);
assertWarnings("At least one multi-field, [sub-field], " +
"was encountered that itself contains a multi-field. Defining multi-fields within a multi-field is deprecated " +
"and will no longer be supported in 8.0. To resolve the issue, all instances of [fields] " +

3
server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
View File

@ -26,7 +26,6 @@ import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@ -284,7 +283,7 @@ public class HighlightBuilderTests extends ESTestCase {
null, null, System::currentTimeMillis, null, null, () -> true, null) {
@Override
public MappedFieldType fieldMapper(String name) {
TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, () -> Lucene.STANDARD_ANALYZER);
TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers());
return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType();
}
};

5
server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java
View File

@ -25,7 +25,6 @@ import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.xcontent.NamedObjectNotFoundException;
@ -148,7 +147,7 @@ public class QueryRescorerBuilderTests extends ESTestCase {
xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null) {
@Override
public MappedFieldType fieldMapper(String name) {
TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, () -> Lucene.STANDARD_ANALYZER);
TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers());
return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType();
}
};
@ -192,7 +191,7 @@ public class QueryRescorerBuilderTests extends ESTestCase {
xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null) {
@Override
public MappedFieldType fieldMapper(String name) {
TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, () -> Lucene.STANDARD_ANALYZER);
TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers());
return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType();
}
};

14
test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
View File

@ -41,6 +41,7 @@ import org.apache.logging.log4j.core.layout.PatternLayout;
import org.apache.logging.log4j.status.StatusConsoleListener;
import org.apache.logging.log4j.status.StatusData;
import org.apache.logging.log4j.status.StatusLogger;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.TestRuleMarkFailure;
@ -95,8 +96,10 @@ import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.indices.analysis.AnalysisModule;
@ -1439,6 +1442,17 @@ public abstract class ESTestCase extends LuceneTestCase {
return elapsed;
}
/**
* Creates an IndexAnalyzers with a single default analyzer
*/
protected IndexAnalyzers createDefaultIndexAnalyzers() {
return new IndexAnalyzers(
Collections.singletonMap("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())),
Collections.emptyMap(),
Collections.emptyMap()
);
}
/**
* Creates an TestAnalysis with all the default analyzers configured.
*/