honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introduce index option named 'index.percolator.map_unmapped_fields_as_string', that handles unmapped fields in percolator queries as type string. 

Closes #9053
Closes #9054
This commit is contained in:
sweetest 2014-12-24 13:31:42 +09:00 committed by Martijn van Groningen
parent a4c92eb67e
commit eaa1674d6d
4 changed files with 57 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
docs/reference/search/percolate.asciidoc
View File

@ -472,3 +472,14 @@ achieve the same result (with way less memory being used).
The delete-by-query api doesn't work to unregister a query, it only deletes the percolate documents from disk. In order
to update the registered queries in memory the index needs be closed and opened.
[float]
=== Forcing unmapped fields to be handled as string
In certain cases it is unknown what kind of percolator queries do get registered and if no field mapping exist for fields
that are referred by percolator queries then adding a percolator query fails. This means the mapping needs to be updated
to have the field with the appropriate settings and then the percolator query can be added. But sometimes it is sufficient
if all unmapped fields are handled as if these were default string fields. In those cases one can configure the
`index.percolator.map_unmapped_fields_as_string` setting to `true` (default to `false`) and then if a field referred in
a percolator query does not exist, it will be handled as a default string field, so adding the percolator query doesn't
fail.

9
src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java
View File

@ -69,6 +69,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
*/
public class PercolatorQueriesRegistry extends AbstractIndexShardComponent implements Closeable{
public final String MAP_UNMAPPED_FIELDS_AS_STRING = "index.percolator.map_unmapped_fields_as_string";
// This is a shard level service, but these below are index level service:
private final IndexQueryParserService queryParserService;
private final MapperService mapperService;
@ -85,6 +87,8 @@ public class PercolatorQueriesRegistry extends AbstractIndexShardComponent imple
private final PercolateTypeListener percolateTypeListener = new PercolateTypeListener();
private final AtomicBoolean realTimePercolatorEnabled = new AtomicBoolean(false);
private boolean mapUnmappedFieldsAsString;
private CloseableThreadLocal<QueryParseContext> cache = new CloseableThreadLocal<QueryParseContext>() {
@Override
protected QueryParseContext initialValue() {
@ -104,6 +108,7 @@ public class PercolatorQueriesRegistry extends AbstractIndexShardComponent imple
this.indexCache = indexCache;
this.indexFieldDataService = indexFieldDataService;
this.shardPercolateService = shardPercolateService;
this.mapUnmappedFieldsAsString = indexSettings.getAsBoolean(MAP_UNMAPPED_FIELDS_AS_STRING, false);
indicesLifecycle.addListener(shardLifecycleListener);
mapperService.addTypeListener(percolateTypeListener);
@ -209,7 +214,11 @@ public class PercolatorQueriesRegistry extends AbstractIndexShardComponent imple
// Query parsing can't introduce new fields in mappings (which happens when registering a percolator query),
// because field type can't be inferred from queries (like document do) so the best option here is to disallow
// the usage of unmapped fields in percolator queries to avoid unexpected behaviour
//
// if index.percolator.map_unmapped_fields_as_string is set to true, query can contain unmapped fields which will be mapped
// as an analyzed string.
context.setAllowUnmappedFields(false);
context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString ? true : false);
return queryParserService.parseInnerQuery(context);
} catch (IOException e) {
throw new QueryParsingException(queryParserService.index(), "Failed to parse", e);

21
src/main/java/org/elasticsearch/index/query/QueryParseContext.java
View File

@ -39,6 +39,7 @@ import org.elasticsearch.common.lucene.search.NoCacheFilter;
import org.elasticsearch.common.lucene.search.NoCacheQuery;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.lucene.search.ResolvableFilter;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AnalysisService;
@ -46,7 +47,11 @@ import org.elasticsearch.index.cache.query.parser.QueryParserCache;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMappers;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MapperBuilders;
import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.index.search.child.CustomQueryWrappingFilter;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.script.ScriptService;
@ -102,6 +107,8 @@ public class QueryParseContext {
private boolean allowUnmappedFields;
private boolean mapUnmappedFieldAsString;
public QueryParseContext(Index index, IndexQueryParserService indexQueryParser) {
this(index, indexQueryParser, false);
}
@ -389,10 +396,6 @@ public class QueryParseContext {
return failIfFieldMappingNotFound(name, indexQueryParser.mapperService.smartName(name, getTypes()));
}
public FieldMapper smartNameFieldMapper(String name) {
return failIfFieldMappingNotFound(name, indexQueryParser.mapperService.smartNameFieldMapper(name, getTypes()));
}
public MapperService.SmartNameObjectMapper smartObjectMapper(String name) {
return indexQueryParser.mapperService.smartNameObjectMapper(name, getTypes());
}
@ -401,9 +404,17 @@ public class QueryParseContext {
this.allowUnmappedFields = allowUnmappedFields;
}
private <T> T failIfFieldMappingNotFound(String name, T fieldMapping) {
public void setMapUnmappedFieldAsString(boolean mapUnmappedFieldAsString) {
this.mapUnmappedFieldAsString = mapUnmappedFieldAsString;
}
private MapperService.SmartNameFieldMappers failIfFieldMappingNotFound(String name, MapperService.SmartNameFieldMappers fieldMapping) {
if (allowUnmappedFields) {
return fieldMapping;
} else if (mapUnmappedFieldAsString){
StringFieldMapper.Builder builder = MapperBuilders.stringField(name);
StringFieldMapper stringFieldMapper = builder.build(new Mapper.BuilderContext(ImmutableSettings.EMPTY, new ContentPath(1)));
return new MapperService.SmartNameFieldMappers(mapperService(), new FieldMappers(stringFieldMapper), null, false);
} else {
Version indexCreatedVersion = indexQueryParser.getIndexCreatedVersion();
if (fieldMapping == null && indexCreatedVersion.onOrAfter(Version.V_1_4_0_Beta1)) {

21
src/test/java/org/elasticsearch/percolator/PercolatorTests.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.ImmutableSettings.Builder;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -2034,5 +2035,25 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
.get();
assertMatchCount(response, 3l);
}
@Test
public void testMapUnmappedFieldAsString() throws IOException{
// If index.percolator.map_unmapped_fields_as_string is set to true, unmapped field is mapped as an analyzed string.
ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder()
.put(indexSettings())
.put("index.percolator.map_unmapped_fields_as_string", true);
assertAcked(prepareCreate("test")
.setSettings(settings));
client().prepareIndex("test", PercolatorService.TYPE_NAME)
.setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject()).get();
logger.info("--> Percolate doc with field1=value");
PercolateResponse response1 = client().preparePercolate()
.setIndices("test").setDocumentType("type")
.setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "value").endObject()))
.execute().actionGet();
assertMatchCount(response1, 1l);
assertThat(response1.getMatches(), arrayWithSize(1));
}
}