honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Ensure that _exists queries on keyword fields use norms when they're available. (#33006)

This commit is contained in:
Julie Tibshirani 2018-08-21 16:33:42 -07:00 committed by GitHub
parent 767c69593c
commit 67b5a83a9a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 45 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
View File

@ -28,6 +28,7 @@ import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.NormsFieldExistsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
@ -166,7 +167,7 @@ public final class KeywordFieldMapper extends FieldMapper {
builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
iterator.remove();
} else if (propName.equals("norms")) {
builder.omitNorms(XContentMapValues.nodeBooleanValue(propNode, "norms") == false);
TypeParsers.parseNorms(builder, name, propNode);
iterator.remove();
} else if (propName.equals("eager_global_ordinals")) {
builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode, "eager_global_ordinals"));
@ -256,8 +257,10 @@ public final class KeywordFieldMapper extends FieldMapper {
public Query existsQuery(QueryShardContext context) {
if (hasDocValues()) {
return new DocValuesFieldExistsQuery(name());
} else {
} else if (omitNorms()) {
return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
} else {
return new NormsFieldExistsQuery(name());
}
}
@ -366,17 +369,19 @@ public final class KeywordFieldMapper extends FieldMapper {
// convert to utf8 only once before feeding postings/dv/stored fields
final BytesRef binaryValue = new BytesRef(value);
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
Field field = new Field(fieldType().name(), binaryValue, fieldType());
fields.add(field);
if (fieldType().hasDocValues() == false && fieldType().omitNorms()) {
createFieldNamesField(context, fields);
}
}
if (fieldType().hasDocValues()) {
fields.add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
} else if (fieldType().stored() || fieldType().indexOptions() != IndexOptions.NONE) {
createFieldNamesField(context, fields);
}
}
@Override
protected String contentType() {
return CONTENT_TYPE;

5
server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
View File

@ -122,8 +122,7 @@ public class TypeParsers {
}
}
public static void parseNorms(FieldMapper.Builder builder, String fieldName, Object propNode,
Mapper.TypeParser.ParserContext parserContext) {
public static void parseNorms(FieldMapper.Builder builder, String fieldName, Object propNode) {
builder.omitNorms(XContentMapValues.nodeBooleanValue(propNode, fieldName + ".norms") == false);
}
@ -140,7 +139,7 @@ public class TypeParsers {
final String propName = entry.getKey();
final Object propNode = entry.getValue();
if ("norms".equals(propName)) {
parseNorms(builder, name, propNode, parserContext);
parseNorms(builder, name, propNode);
iterator.remove();
}
}

16
server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
View File

@ -321,11 +321,16 @@ public class KeywordFieldMapperTests extends ESSingleNodeTestCase {
public void testEnableNorms() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", "keyword").field("norms", true).endObject().endObject()
.endObject().endObject());
.startObject("properties")
.startObject("field")
.field("type", "keyword")
.field("doc_values", false)
.field("norms", true)
.endObject()
.endObject()
.endObject().endObject());
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, mapper.mappingSource().toString());
ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", BytesReference
@ -336,8 +341,11 @@ public class KeywordFieldMapperTests extends ESSingleNodeTestCase {
XContentType.JSON));
IndexableField[] fields = doc.rootDoc().getFields("field");
assertEquals(2, fields.length);
assertEquals(1, fields.length);
assertFalse(fields[0].fieldType().omitNorms());
IndexableField[] fieldNamesFields = doc.rootDoc().getFields(FieldNamesFieldMapper.NAME);
assertEquals(0, fieldNamesFields.length);
}
public void testNormalizer() throws IOException {

22
server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.index.mapper;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenFilter;
@ -28,9 +27,11 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.NormsFieldExistsQuery;
import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.Lucene;
@ -132,6 +133,23 @@ public class KeywordFieldTypeTests extends FieldTypeTestCase {
assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
}
public void testExistsQuery() {
MappedFieldType ft = createDefaultFieldType();
ft.setName("field");
ft.setHasDocValues(true);
ft.setOmitNorms(true);
assertEquals(new DocValuesFieldExistsQuery("field"), ft.existsQuery(null));
ft.setHasDocValues(false);
ft.setOmitNorms(false);
assertEquals(new NormsFieldExistsQuery("field"), ft.existsQuery(null));
ft.setHasDocValues(false);
ft.setOmitNorms(true);
assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null));
}
public void testRegexpQuery() {
MappedFieldType ft = createDefaultFieldType();
ft.setName("field");