Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-11023: Added EnumFieldType, a non-Trie-based version of EnumField, and deprecated EnumField in favor of EnumFieldType.

This commit is contained in:
Steve Rowe 2017-08-04 19:32:46 -04:00
parent c1d28c3ece
commit 9627d1db5d
20 changed files with 1038 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
solr/CHANGES.txt
View File

@ -263,6 +263,8 @@ Upgrading from Solr 6.x
* All deperated methods of ClusterState (except getZkClusterStateVersion())
have been removed. Use DocCollection methods instead.
* SOLR-11023: EnumField has been deprecated in favor of new EnumFieldType.
New Features
----------------------
* SOLR-9857, SOLR-9858: Collect aggregated metrics from nodes and shard leaders in overseer. (ab)
@ -618,6 +620,9 @@ Other Changes
* SOLR-11178: Change error handling in AutoScalingHandler to be consistent w/ other APIs (noble)
* SOLR-11023: Added EnumFieldType, a non-Trie-based version of EnumField, and deprecated EnumField
in favor of EnumFieldType. (hossman, Steve Rowe)
================== 6.7.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

2
solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
View File

@ -78,7 +78,7 @@ public class StatsValuesFactory {
return statsValue;
} else if (StrField.class.isInstance(fieldType)) {
return new StringStatsValues(statsField);
} else if (sf.getType().getClass().equals(EnumField.class)) {
} else if (AbstractEnumField.class.isInstance(fieldType)) {
return new EnumStatsValues(statsField);
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,

311
solr/core/src/java/org/apache/solr/schema/AbstractEnumField.java Normal file
View File

@ -0,0 +1,311 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.schema;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.EnumFieldSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
/***
* Abstract Field type for support of string values with custom sort order.
*/
public abstract class AbstractEnumField extends PrimitiveFieldType {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected EnumMapping enumMapping;
@Override
protected void init(IndexSchema schema, Map<String, String> args) {
super.init(schema, args);
enumMapping = new EnumMapping(schema, this, args);
}
public EnumMapping getEnumMapping() {
return enumMapping;
}
/**
* Models all the info contained in an enums config XML file
* @lucene.internal
*/
public static final class EnumMapping {
public static final String PARAM_ENUMS_CONFIG = "enumsConfig";
public static final String PARAM_ENUM_NAME = "enumName";
public static final Integer DEFAULT_VALUE = -1;
public final Map<String, Integer> enumStringToIntMap;
public final Map<Integer, String> enumIntToStringMap;
protected final String enumsConfigFile;
protected final String enumName;
/**
* Takes in a FieldType and the initArgs Map used for that type, removing the keys
* that specify the enum.
*
* @param schema for opening resources
* @param fieldType Used for logging or error messages
* @param args the init args to comsume the enum name + config file from
*/
public EnumMapping(IndexSchema schema, FieldType fieldType, Map<String, String> args) {
final String ftName = fieldType.getTypeName();
// NOTE: ghosting member variables for most of constructor
final Map<String, Integer> enumStringToIntMap = new HashMap<>();
final Map<Integer, String> enumIntToStringMap = new HashMap<>();
enumsConfigFile = args.get(PARAM_ENUMS_CONFIG);
if (enumsConfigFile == null) {
throw new SolrException(SolrException.ErrorCode.NOT_FOUND,
ftName + ": No enums config file was configured.");
}
enumName = args.get(PARAM_ENUM_NAME);
if (enumName == null) {
throw new SolrException(SolrException.ErrorCode.NOT_FOUND,
ftName + ": No enum name was configured.");
}
InputStream is = null;
try {
is = schema.getResourceLoader().openResource(enumsConfigFile);
final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
try {
final Document doc = dbf.newDocumentBuilder().parse(is);
final XPathFactory xpathFactory = XPathFactory.newInstance();
final XPath xpath = xpathFactory.newXPath();
final String xpathStr = String.format(Locale.ROOT, "/enumsConfig/enum[@name='%s']", enumName);
final NodeList nodes = (NodeList) xpath.evaluate(xpathStr, doc, XPathConstants.NODESET);
final int nodesLength = nodes.getLength();
if (nodesLength == 0) {
String exceptionMessage = String.format
(Locale.ENGLISH, "%s: No enum configuration found for enum '%s' in %s.",
ftName, enumName, enumsConfigFile);
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, exceptionMessage);
}
if (nodesLength > 1) {
if (log.isWarnEnabled())
log.warn("{}: More than one enum configuration found for enum '{}' in {}. The last one was taken.",
ftName, enumName, enumsConfigFile);
}
final Node enumNode = nodes.item(nodesLength - 1);
final NodeList valueNodes = (NodeList) xpath.evaluate("value", enumNode, XPathConstants.NODESET);
for (int i = 0; i < valueNodes.getLength(); i++) {
final Node valueNode = valueNodes.item(i);
final String valueStr = valueNode.getTextContent();
if ((valueStr == null) || (valueStr.length() == 0)) {
final String exceptionMessage = String.format
(Locale.ENGLISH, "%s: A value was defined with an no value in enum '%s' in %s.",
ftName, enumName, enumsConfigFile);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, exceptionMessage);
}
if (enumStringToIntMap.containsKey(valueStr)) {
final String exceptionMessage = String.format
(Locale.ENGLISH, "%s: A duplicated definition was found for value '%s' in enum '%s' in %s.",
ftName, valueStr, enumName, enumsConfigFile);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, exceptionMessage);
}
enumIntToStringMap.put(i, valueStr);
enumStringToIntMap.put(valueStr, i);
}
}
catch (ParserConfigurationException | XPathExpressionException | SAXException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
ftName + ": Error parsing enums config.", e);
}
}
catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
ftName + ": Error while opening enums config.", e);
} finally {
try {
if (is != null) {
is.close();
}
}
catch (IOException e) {
e.printStackTrace();
}
}
if ((enumStringToIntMap.size() == 0) || (enumIntToStringMap.size() == 0)) {
String exceptionMessage = String.format
(Locale.ENGLISH, "%s: Invalid configuration was defined for enum '%s' in %s.",
ftName, enumName, enumsConfigFile);
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, exceptionMessage);
}
this.enumStringToIntMap = Collections.unmodifiableMap(enumStringToIntMap);
this.enumIntToStringMap = Collections.unmodifiableMap(enumIntToStringMap);
args.remove(PARAM_ENUMS_CONFIG);
args.remove(PARAM_ENUM_NAME);
}
/**
* Converting the (internal) integer value (indicating the sort order) to string (displayed) value
* @param intVal integer value
* @return string value
*/
public String intValueToStringValue(Integer intVal) {
if (intVal == null)
return null;
final String enumString = enumIntToStringMap.get(intVal);
if (enumString != null)
return enumString;
// can't find matching enum name - return DEFAULT_VALUE.toString()
return DEFAULT_VALUE.toString();
}
/**
* Converting the string (displayed) value (internal) to integer value (indicating the sort order)
* @param stringVal string value
* @return integer value
*/
public Integer stringValueToIntValue(String stringVal) {
if (stringVal == null)
return null;
Integer intValue;
final Integer enumInt = enumStringToIntMap.get(stringVal);
if (enumInt != null) //enum int found for string
return enumInt;
//enum int not found for string
intValue = tryParseInt(stringVal);
if (intValue == null) //not Integer
intValue = DEFAULT_VALUE;
final String enumString = enumIntToStringMap.get(intValue);
if (enumString != null) //has matching string
return intValue;
return DEFAULT_VALUE;
}
private static Integer tryParseInt(String valueStr) {
Integer intValue = null;
try {
intValue = Integer.parseInt(valueStr);
}
catch (NumberFormatException e) {
}
return intValue;
}
}
@Override
public EnumFieldValue toObject(IndexableField f) {
Integer intValue = null;
String stringValue = null;
final Number val = f.numericValue();
if (val != null) {
intValue = val.intValue();
stringValue = enumMapping.intValueToStringValue(intValue);
}
return new EnumFieldValue(intValue, stringValue);
}
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
final Object missingValue = Integer.MIN_VALUE;
SortField sf = new SortField(field.getName(), SortField.Type.INT, top);
sf.setMissingValue(missingValue);
return sf;
}
@Override
public ValueSource getValueSource(SchemaField field, QParser qparser) {
field.checkFieldCacheSource();
return new EnumFieldSource(field.getName(), enumMapping.enumIntToStringMap, enumMapping.enumStringToIntMap);
}
@Override
public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException {
final Number val = f.numericValue();
if (val == null) {
writer.writeNull(name);
return;
}
final String readableValue = enumMapping.intValueToStringValue(val.intValue());
writer.writeStr(name, readableValue, true);
}
@Override
public boolean isTokenized() {
return false;
}
@Override
public NumberType getNumberType() {
return NumberType.INTEGER;
}
@Override
public String readableToIndexed(String val) {
if (val == null)
return null;
final BytesRefBuilder bytes = new BytesRefBuilder();
readableToIndexed(val, bytes);
return bytes.get().utf8ToString();
}
@Override
public String toInternal(String val) {
return readableToIndexed(val);
}
@Override
public String toExternal(IndexableField f) {
final Number val = f.numericValue();
if (val == null)
return null;
return enumMapping.intValueToStringValue(val.intValue());
}
}

311
solr/core/src/java/org/apache/solr/schema/EnumField.java
View File

@ -16,21 +16,11 @@
*/
package org.apache.solr.schema;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
@ -40,154 +30,29 @@ import org.apache.solr.legacy.LegacyIntField;
import org.apache.solr.legacy.LegacyNumericRangeQuery;
import org.apache.solr.legacy.LegacyNumericType;
import org.apache.solr.legacy.LegacyNumericUtils;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.EnumFieldSource;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
/***
/**
* Field type for support of string values with custom sort order.
* @deprecated use {@link EnumFieldType} instead.
*/
public class EnumField extends PrimitiveFieldType {
@Deprecated
public class EnumField extends AbstractEnumField {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected static final String PARAM_ENUMS_CONFIG = "enumsConfig";
protected static final String PARAM_ENUM_NAME = "enumName";
protected static final Integer DEFAULT_VALUE = -1;
protected static final int DEFAULT_PRECISION_STEP = Integer.MAX_VALUE;
protected Map<String, Integer> enumStringToIntMap = new HashMap<>();
protected Map<Integer, String> enumIntToStringMap = new HashMap<>();
protected String enumsConfigFile;
protected String enumName;
/**
* {@inheritDoc}
*/
@Override
protected void init(IndexSchema schema, Map<String, String> args) {
super.init(schema, args);
enumsConfigFile = args.get(PARAM_ENUMS_CONFIG);
if (enumsConfigFile == null) {
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No enums config file was configured.");
}
enumName = args.get(PARAM_ENUM_NAME);
if (enumName == null) {
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No enum name was configured.");
}
InputStream is = null;
try {
is = schema.getResourceLoader().openResource(enumsConfigFile);
final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
try {
final Document doc = dbf.newDocumentBuilder().parse(is);
final XPathFactory xpathFactory = XPathFactory.newInstance();
final XPath xpath = xpathFactory.newXPath();
final String xpathStr = String.format(Locale.ROOT, "/enumsConfig/enum[@name='%s']", enumName);
final NodeList nodes = (NodeList) xpath.evaluate(xpathStr, doc, XPathConstants.NODESET);
final int nodesLength = nodes.getLength();
if (nodesLength == 0) {
String exceptionMessage = String.format(Locale.ENGLISH, "No enum configuration found for enum '%s' in %s.",
enumName, enumsConfigFile);
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, exceptionMessage);
}
if (nodesLength > 1) {
if (log.isWarnEnabled())
log.warn("More than one enum configuration found for enum '{}' in {}. The last one was taken.", enumName, enumsConfigFile);
}
final Node enumNode = nodes.item(nodesLength - 1);
final NodeList valueNodes = (NodeList) xpath.evaluate("value", enumNode, XPathConstants.NODESET);
for (int i = 0; i < valueNodes.getLength(); i++) {
final Node valueNode = valueNodes.item(i);
final String valueStr = valueNode.getTextContent();
if ((valueStr == null) || (valueStr.length() == 0)) {
final String exceptionMessage = String.format(Locale.ENGLISH, "A value was defined with an no value in enum '%s' in %s.",
enumName, enumsConfigFile);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, exceptionMessage);
}
if (enumStringToIntMap.containsKey(valueStr)) {
final String exceptionMessage = String.format(Locale.ENGLISH, "A duplicated definition was found for value '%s' in enum '%s' in %s.",
valueStr, enumName, enumsConfigFile);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, exceptionMessage);
}
enumIntToStringMap.put(i, valueStr);
enumStringToIntMap.put(valueStr, i);
}
}
catch (ParserConfigurationException | XPathExpressionException | SAXException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing enums config.", e);
}
}
catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error while opening enums config.", e);
} finally {
try {
if (is != null) {
is.close();
}
}
catch (IOException e) {
e.printStackTrace();
}
}
if ((enumStringToIntMap.size() == 0) || (enumIntToStringMap.size() == 0)) {
String exceptionMessage = String.format(Locale.ENGLISH, "Invalid configuration was defined for enum '%s' in %s.",
enumName, enumsConfigFile);
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, exceptionMessage);
}
args.remove(PARAM_ENUMS_CONFIG);
args.remove(PARAM_ENUM_NAME);
}
/**
* {@inheritDoc}
*/
@Override
public EnumFieldValue toObject(IndexableField f) {
Integer intValue = null;
String stringValue = null;
final Number val = f.numericValue();
if (val != null) {
intValue = val.intValue();
stringValue = intValueToStringValue(intValue);
}
return new EnumFieldValue(intValue, stringValue);
}
/**
* {@inheritDoc}
*/
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
final Object missingValue = Integer.MIN_VALUE;
SortField sf = new SortField(field.getName(), SortField.Type.INT, top);
sf.setMissingValue(missingValue);
return sf;
}
@Override
public Type getUninversionType(SchemaField sf) {
if (sf.multiValued()) {
@ -197,53 +62,10 @@ public class EnumField extends PrimitiveFieldType {
}
}
/**
* {@inheritDoc}
*/
@Override
public ValueSource getValueSource(SchemaField field, QParser qparser) {
field.checkFieldCacheSource();
return new EnumFieldSource(field.getName(), enumIntToStringMap, enumStringToIntMap);
}
/**
* {@inheritDoc}
*/
@Override
public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException {
final Number val = f.numericValue();
if (val == null) {
writer.writeNull(name);
return;
}
final String readableValue = intValueToStringValue(val.intValue());
writer.writeStr(name, readableValue, true);
}
/**
* {@inheritDoc}
*/
@Override
public boolean isTokenized() {
return false;
}
/**
* {@inheritDoc}
*/
@Override
public NumberType getNumberType() {
return NumberType.INTEGER;
}
/**
* {@inheritDoc}
*/
@Override
public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
Integer minValue = stringValueToIntValue(min);
Integer maxValue = stringValueToIntValue(max);
Integer minValue = enumMapping.stringValueToIntValue(min);
Integer maxValue = enumMapping.stringValueToIntValue(max);
if (field.multiValued() && field.hasDocValues() && !field.indexed()) {
// for the multi-valued dv-case, the default rangeimpl over toInternal is correct
@ -277,90 +99,42 @@ public class EnumField extends PrimitiveFieldType {
return query;
}
/**
* {@inheritDoc}
*/
@Override
public String readableToIndexed(String val) {
if (val == null)
return null;
final BytesRefBuilder bytes = new BytesRefBuilder();
readableToIndexed(val, bytes);
return bytes.get().utf8ToString();
}
/**
* {@inheritDoc}
*/
@Override
public void readableToIndexed(CharSequence val, BytesRefBuilder result) {
final String s = val.toString();
if (s == null)
return;
final Integer intValue = stringValueToIntValue(s);
final Integer intValue = enumMapping.stringValueToIntValue(s);
LegacyNumericUtils.intToPrefixCoded(intValue, 0, result);
}
/**
* {@inheritDoc}
*/
@Override
public String toInternal(String val) {
return readableToIndexed(val);
}
/**
* {@inheritDoc}
*/
@Override
public String toExternal(IndexableField f) {
final Number val = f.numericValue();
if (val == null)
return null;
return intValueToStringValue(val.intValue());
}
/**
* {@inheritDoc}
*/
@Override
public String indexedToReadable(String indexedForm) {
if (indexedForm == null)
return null;
final BytesRef bytesRef = new BytesRef(indexedForm);
final Integer intValue = LegacyNumericUtils.prefixCodedToInt(bytesRef);
return intValueToStringValue(intValue);
return enumMapping.intValueToStringValue(intValue);
}
/**
* {@inheritDoc}
*/
@Override
public CharsRef indexedToReadable(BytesRef input, CharsRefBuilder output) {
final Integer intValue = LegacyNumericUtils.prefixCodedToInt(input);
final String stringValue = intValueToStringValue(intValue);
final String stringValue = enumMapping.intValueToStringValue(intValue);
output.grow(stringValue.length());
output.setLength(stringValue.length());
stringValue.getChars(0, output.length(), output.chars(), 0);
return output.get();
}
/**
* {@inheritDoc}
*/
@Override
public EnumFieldValue toObject(SchemaField sf, BytesRef term) {
final Integer intValue = LegacyNumericUtils.prefixCodedToInt(term);
final String stringValue = intValueToStringValue(intValue);
final String stringValue = enumMapping.intValueToStringValue(intValue);
return new EnumFieldValue(intValue, stringValue);
}
/**
* {@inheritDoc}
*/
@Override
public String storedToIndexed(IndexableField f) {
final Number val = f.numericValue();
@ -371,9 +145,6 @@ public class EnumField extends PrimitiveFieldType {
return bytes.get().utf8ToString();
}
/**
* {@inheritDoc}
*/
@Override
public IndexableField createField(SchemaField field, Object value) {
final boolean indexed = field.indexed();
@ -385,8 +156,8 @@ public class EnumField extends PrimitiveFieldType {
log.trace("Ignoring unindexed/unstored field: " + field);
return null;
}
final Integer intValue = stringValueToIntValue(value.toString());
if (intValue == null || intValue.equals(DEFAULT_VALUE)) {
final Integer intValue = enumMapping.stringValueToIntValue(value.toString());
if (intValue == null || intValue.equals(EnumMapping.DEFAULT_VALUE)) {
String exceptionMessage = String.format(Locale.ENGLISH, "Unknown value for enum field: %s, value: %s",
field.getName(), value.toString());
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, exceptionMessage);
@ -408,9 +179,6 @@ public class EnumField extends PrimitiveFieldType {
return new LegacyIntField(field.getName(), intValue.intValue(), newType);
}
/**
* {@inheritDoc}
*/
@Override
public List<IndexableField> createFields(SchemaField sf, Object value) {
if (sf.hasDocValues()) {
@ -420,7 +188,7 @@ public class EnumField extends PrimitiveFieldType {
if (sf.multiValued()) {
BytesRefBuilder bytes = new BytesRefBuilder();
readableToIndexed(stringValueToIntValue(value.toString()).toString(), bytes);
readableToIndexed(enumMapping.stringValueToIntValue(value.toString()).toString(), bytes);
fields.add(new SortedSetDocValuesField(sf.getName(), bytes.toBytesRef()));
} else {
final long bits = field.numericValue().intValue();
@ -431,57 +199,4 @@ public class EnumField extends PrimitiveFieldType {
return Collections.singletonList(createField(sf, value));
}
}
/**
* Converting the (internal) integer value (indicating the sort order) to string (displayed) value
* @param intVal integer value
* @return string value
*/
public String intValueToStringValue(Integer intVal) {
if (intVal == null)
return null;
final String enumString = enumIntToStringMap.get(intVal);
if (enumString != null)
return enumString;
// can't find matching enum name - return DEFAULT_VALUE.toString()
return DEFAULT_VALUE.toString();
}
/**
* Converting the string (displayed) value (internal) to integer value (indicating the sort order)
* @param stringVal string value
* @return integer value
*/
public Integer stringValueToIntValue(String stringVal) {
if (stringVal == null)
return null;
Integer intValue;
final Integer enumInt = enumStringToIntMap.get(stringVal);
if (enumInt != null) //enum int found for string
return enumInt;
//enum int not found for string
intValue = tryParseInt(stringVal);
if (intValue == null) //not Integer
intValue = DEFAULT_VALUE;
final String enumString = enumIntToStringMap.get(intValue);
if (enumString != null) //has matching string
return intValue;
return DEFAULT_VALUE;
}
private static Integer tryParseInt(String valueStr) {
Integer intValue = null;
try {
intValue = Integer.parseInt(valueStr);
}
catch (NumberFormatException e) {
}
return intValue;
}
}

213
solr/core/src/java/org/apache/solr/schema/EnumFieldType.java Normal file
View File

@ -0,0 +1,213 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.schema;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.MultiValuedIntFieldSource;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortedNumericSelector;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.NumericUtils;
import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.SolrException;
import org.apache.solr.search.QParser;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Field type for support of string values with custom sort order.
*/
public class EnumFieldType extends AbstractEnumField {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@Override
public Type getUninversionType(SchemaField sf) {
return null;
}
@Override
public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
Integer minValue = enumMapping.stringValueToIntValue(min);
Integer maxValue = enumMapping.stringValueToIntValue(max);
if (field.indexed()) {
BytesRef minBytes = null;
if (min != null) {
byte[] bytes = new byte[Integer.BYTES];
NumericUtils.intToSortableBytes(minValue, bytes, 0);
minBytes = new BytesRef(bytes);
}
BytesRef maxBytes = null;
if (max != null) {
byte[] bytes = new byte[Integer.BYTES];
NumericUtils.intToSortableBytes(maxValue, bytes, 0);
maxBytes = new BytesRef(bytes);
}
return new TermRangeQuery(field.getName(), minBytes, maxBytes, minInclusive, maxInclusive);
} else {
long lowerValue = Long.MIN_VALUE;
long upperValue = Long.MAX_VALUE;
if (minValue != null) {
lowerValue = minValue.longValue();
if (minInclusive == false) {
++lowerValue;
}
}
if (maxValue != null) {
upperValue = maxValue.longValue();
if (maxInclusive == false) {
--upperValue;
}
}
if (field.multiValued()) {
return new ConstantScoreQuery(SortedNumericDocValuesField.newSlowRangeQuery
(field.getName(), lowerValue, upperValue));
} else {
return new ConstantScoreQuery(NumericDocValuesField.newSlowRangeQuery
(field.getName(), lowerValue, upperValue));
}
}
}
@Override
public void readableToIndexed(CharSequence val, BytesRefBuilder result) {
final String s = val.toString();
if (s == null)
return;
result.grow(Integer.BYTES);
result.setLength(Integer.BYTES);
final Integer intValue = enumMapping.stringValueToIntValue(s);
NumericUtils.intToSortableBytes(intValue, result.bytes(), 0);
}
@Override
public String indexedToReadable(String indexedForm) {
if (indexedForm == null)
return null;
final BytesRef bytesRef = new BytesRef(indexedForm);
final Integer intValue = NumericUtils.sortableBytesToInt(bytesRef.bytes, 0);
return enumMapping.intValueToStringValue(intValue);
}
@Override
public CharsRef indexedToReadable(BytesRef input, CharsRefBuilder output) {
final Integer intValue = NumericUtils.sortableBytesToInt(input.bytes, 0);
final String stringValue = enumMapping.intValueToStringValue(intValue);
output.grow(stringValue.length());
output.setLength(stringValue.length());
stringValue.getChars(0, output.length(), output.chars(), 0);
return output.get();
}
@Override
public EnumFieldValue toObject(SchemaField sf, BytesRef term) {
final Integer intValue = NumericUtils.sortableBytesToInt(term.bytes, 0);
final String stringValue = enumMapping.intValueToStringValue(intValue);
return new EnumFieldValue(intValue, stringValue);
}
@Override
public String storedToIndexed(IndexableField f) {
final Number val = f.numericValue();
if (val == null)
return null;
final BytesRefBuilder bytes = new BytesRefBuilder();
bytes.grow(Integer.BYTES);
bytes.setLength(Integer.BYTES);
NumericUtils.intToSortableBytes(val.intValue(), bytes.bytes(), 0);
return bytes.get().utf8ToString();
}
@Override
public IndexableField createField(SchemaField field, Object value) {
final Integer intValue = enumMapping.stringValueToIntValue(value.toString());
if (intValue == null || intValue.equals(EnumMapping.DEFAULT_VALUE)) {
String exceptionMessage = String.format(Locale.ENGLISH, "Unknown value for enum field: %s, value: %s",
field.getName(), value.toString());
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, exceptionMessage);
}
org.apache.lucene.document.FieldType newType = new org.apache.lucene.document.FieldType();
newType.setTokenized(false);
newType.setStored(field.stored());
newType.setOmitNorms(field.omitNorms());
newType.setIndexOptions(field.indexOptions());
newType.setStoreTermVectors(field.storeTermVector());
newType.setStoreTermVectorOffsets(field.storeTermOffsets());
newType.setStoreTermVectorPositions(field.storeTermPositions());
newType.setStoreTermVectorPayloads(field.storeTermPayloads());
byte[] bytes = new byte[Integer.BYTES];
NumericUtils.intToSortableBytes(intValue, bytes, 0);
return new Field(field.getName(), bytes, newType) {
@Override public Number numericValue() {
return NumericUtils.sortableBytesToInt(((BytesRef)fieldsData).bytes, 0);
}
};
}
@Override
public List<IndexableField> createFields(SchemaField sf, Object value) {
if ( ! sf.hasDocValues()) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
getClass().getSimpleName() + " requires docValues=\"true\".");
}
final IndexableField field = createField(sf, value);
final List<IndexableField> fields = new ArrayList<>();
fields.add(field);
final long longValue = field.numericValue().longValue();
if (sf.multiValued()) {
fields.add(new SortedNumericDocValuesField(sf.getName(), longValue));
} else {
fields.add(new NumericDocValuesField(sf.getName(), longValue));
}
return fields;
}
@Override
public final ValueSource getSingleValueSource(MultiValueSelector choice, SchemaField field, QParser parser) {
if ( ! field.multiValued()) { // trivial base case
return getValueSource(field, parser); // single value matches any selector
}
SortedNumericSelector.Type selectorType = choice.getSortedNumericSelectorType();
if (null == selectorType) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
choice.toString() + " is not a supported option for picking a single value"
+ " from the multivalued field: " + field.getName() +
" (type: " + this.getTypeName() + ")");
}
return new MultiValuedIntFieldSource(field.getName(), selectorType);
}
}

15
solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
View File

@ -56,7 +56,7 @@ import org.apache.lucene.util.NumericUtils;
import org.apache.solr.common.SolrDocumentBase;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.schema.BoolField;
import org.apache.solr.schema.EnumField;
import org.apache.solr.schema.AbstractEnumField;
import org.apache.solr.schema.NumberType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TrieDateField;
@ -465,8 +465,8 @@ public class SolrDocumentFetcher {
newVal = Double.longBitsToDouble(val);
} else if (schemaField.getType() instanceof TrieDateField) {
newVal = new Date(val);
} else if (schemaField.getType() instanceof EnumField) {
newVal = ((EnumField) schemaField.getType()).intValueToStringValue(val.intValue());
} else if (schemaField.getType() instanceof AbstractEnumField) {
newVal = ((AbstractEnumField)schemaField.getType()).getEnumMapping().intValueToStringValue(val.intValue());
}
}
doc.addField(fieldName, newVal);
@ -501,7 +501,7 @@ public class SolrDocumentFetcher {
break;
case SORTED_NUMERIC:
final SortedNumericDocValues numericDv = leafReader.getSortedNumericDocValues(fieldName);
NumberType type = schemaField.getType().getNumberType();
final NumberType type = schemaField.getType().getNumberType();
if (numericDv != null) {
if (numericDv.advance(localId) == localId) {
final List<Object> outValues = new ArrayList<Object>(numericDv.docValueCount());
@ -509,7 +509,12 @@ public class SolrDocumentFetcher {
long number = numericDv.nextValue();
switch (type) {
case INTEGER:
outValues.add((int)number);
final int raw = (int)number;
if (schemaField.getType() instanceof AbstractEnumField) {
outValues.add(((AbstractEnumField)schemaField.getType()).getEnumMapping().intValueToStringValue(raw));
} else {
outValues.add(raw);
}
break;
case LONG:
outValues.add(number);

34
solr/core/src/test-files/solr/collection1/conf/bad-schema-enums.xml Normal file
View File

@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<schema name="bad-schema-enums" version="1.6">
<field name="id" type="string" indexed="true" stored="true" required="true"/>
<field name="_version_" type="long" indexed="true" stored="true" multiValued="false"/>
<!-- Test EnumFieldType -->
<!-- Begin bad stuff: EnumFieldType requires docValues -->
<field name="severity" type="severityType" indexed="${solr.tests.EnumFieldTest.indexed}" stored="true" multiValued="false" docValues="false"/>
<!-- End bad stuff -->
<uniqueKey>id</uniqueKey>
<!-- note: you cannot change the order/existing values in enum without reindexing.
but you can always add new values to the end. -->
<fieldType name="severityType" class="solr.EnumFieldType" enumsConfig="enumsConfig.xml" enumName="severity"/>
<fieldType name="string" class="solr.StrField"/>
<fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
</schema>

19
solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
View File

@ -27,7 +27,26 @@
<value>Low</value>
<value>Medium</value>
<value>High</value>
<!-- we define a bunch of "filler" enum values to ensure "Critical" gets a value
of "11" so we can sanity check that sorting and range queries don't use lexical ordering
Low(1) < High(3) < Critical(11)
-->
<value>x4</value>
<value>x5</value>
<value>x6</value>
<value>x7</value>
<value>x8</value>
<value>x9</value>
<value>x10</value>
<value>Critical</value>
<!-- More "filler" enum values to exceed SolrQueryParser.TERMS_QUERY_THRESHOLD, to generate set queries -->
<value>x12</value>
<value>x13</value>
<value>x14</value>
<value>x15</value>
<value>x16</value>
<value>x17</value>
<value>x18</value>
</enum>
</enumsConfig>

8
solr/core/src/test-files/solr/collection1/conf/schema-enums.xml
View File

@ -18,9 +18,9 @@
<schema name="tiny" version="1.1">
<field name="id" type="string" indexed="true" stored="true" required="true"/>
<field name="_version_" type="long" indexed="true" stored="true" multiValued="false"/>
<!-- Test EnumField -->
<field name="severity" type="severityType" indexed="true" stored="true" multiValued="false"/>
<field name="severity_dv" type="severityType" indexed="true" stored="true" multiValued="false" docValues="true"/>
<!-- Test EnumField and EnumFieldType -->
<field name="severity" type="severityType" indexed="${solr.tests.EnumFieldTest.indexed}" stored="true" multiValued="false" docValues="${solr.tests.numeric.dv}"/>
<field name="severity_mv" type="severityType" indexed="${solr.tests.EnumFieldTest.indexed}" stored="true" multiValued="true" docValues="${solr.tests.numeric.dv}"/>
<field name="text" type="text" indexed="true" stored="true" multiValued="true"/>
<uniqueKey>id</uniqueKey>
@ -32,7 +32,7 @@
</fieldType>
<!-- note: you cannot change the order/existing values in enum without reindexing.
but you can always add new values to the end. -->
<fieldType name="severityType" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="severity"/>
<fieldType name="severityType" class="${solr.tests.EnumFieldType}" enumsConfig="enumsConfig.xml" enumName="severity"/>
<fieldType name="string" class="solr.StrField"/>
<fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>

2
solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml
View File

@ -26,7 +26,7 @@
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
<fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
<fieldType name="date" class="${solr.tests.DateFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0"/>
<fieldType name="enumField" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="severity"/>
<fieldType name="enumField" class="${solr.tests.EnumFieldType}" enumsConfig="enumsConfig.xml" enumName="severity"/>
<field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>

21
solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml
View File

@ -109,10 +109,9 @@ NOTE: Tests expect every field in this schema to be sortable.
<field name="enum" type="enum"/>
<field name="enum_last" type="enum_last"/>
<field name="enum_first" type="enum_first"/>
<!-- EnumField incorrectly disallows missing DocValues - see SOLR-5927 -->
<!-- <field name="enum_dv" type="enum_dv" /> -->
<!-- <field name="enum_dv_last" type="enum_dv_last" /> -->
<!-- <field name="enum_dv_first" type="enum_dv_first" /> -->
<field name="enum_dv" type="enum_dv" />
<field name="enum_dv_last" type="enum_dv_last" />
<field name="enum_dv_first" type="enum_dv_first" />
<!-- ensure function sorts don't mistakenly get interpreted as field sorts
https://issues.apache.org/jira/browse/SOLR-5354?focusedCommentId=13835891&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-13835891
@ -185,10 +184,9 @@ NOTE: Tests expect every field in this schema to be sortable.
<copyField source="enum" dest="enum_last"/>
<copyField source="enum" dest="enum_first"/>
<!-- EnumField incorrectly disallows missing DocValues - see SOLR-5927 -->
<!-- <copyField source="enum" dest="enum_dv" /> -->
<!-- <copyField source="enum" dest="enum_dv_last" /> -->
<!-- <copyField source="enum" dest="enum_dv_first" /> -->
<copyField source="enum" dest="enum_dv" />
<copyField source="enum" dest="enum_dv_last" />
<copyField source="enum" dest="enum_dv_first" />
<fieldType name="str" class="solr.StrField" stored="true" indexed="true"/>
@ -309,9 +307,8 @@ NOTE: Tests expect every field in this schema to be sortable.
sortMissingLast="true"/>
<fieldType name="enum_first" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="severity"
sortMissingFirst="true"/>
<!-- EnumField incorrectly disallows missing DocValues - see SOLR-5927 -->
<!-- <fieldType name="enum_dv" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="severity" docValues="true"/> -->
<!-- <fieldType name="enum_dv_last" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="severity" docValues="true" sortMissingLast="true"/> -->
<!-- <fieldType name="enum_dv_first" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="severity" docValues="true" sortMissingFirst="true"/> -->
<fieldType name="enum_dv" class="${solr.tests.EnumFieldType}" enumsConfig="enumsConfig.xml" enumName="severity" docValues="true"/>
<fieldType name="enum_dv_last" class="${solr.tests.EnumFieldType}" enumsConfig="enumsConfig.xml" enumName="severity" docValues="true" sortMissingLast="true"/>
<fieldType name="enum_dv_first" class="${solr.tests.EnumFieldType}" enumsConfig="enumsConfig.xml" enumName="severity" docValues="true" sortMissingFirst="true"/>
</schema>

2
solr/core/src/test-files/solr/collection1/conf/schema.xml
View File

@ -506,7 +506,7 @@
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
</analyzer>
</fieldType>
<fieldType name="severityType" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="severity"/>
<fieldType name="severityType" class="${solr.tests.EnumFieldType}" enumsConfig="enumsConfig.xml" enumName="severity"/>
<field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
<field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>

8
solr/core/src/test-files/solr/collection1/conf/schema11.xml
View File

@ -295,8 +295,8 @@ valued. -->
</fieldType>
<!-- EnumType -->
<fieldType name="severityType" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="severity"/>
<!-- Enum type -->
<fieldType name="severityType" class="${solr.tests.EnumFieldType}" enumsConfig="enumsConfig.xml" enumName="severity"/>
<!-- Valid attributes for fields:
name: mandatory - the name for the field
@ -340,8 +340,8 @@ valued. -->
<field name="cat_length" type="text_length" indexed="true" stored="true" multiValued="true"/>
<!-- EnumType -->
<field name="severity" type="severityType" indexed="true" stored="true" multiValued="false"/>
<!-- Enum type -->
<field name="severity" type="severityType" docValues="true" indexed="true" stored="true" multiValued="false"/>
<!-- Dynamic field definitions. If a field name is not found, dynamicFields
will be used if the name matches any of the patterns.

2
solr/core/src/test/org/apache/solr/TestDistributedSearch.java
View File

@ -1062,7 +1062,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
rsp.getFieldStatsInfo().get(fieldName).getMin());
query("q", "*:*", "stats", "true", "stats.field", fieldName,
StatsParams.STATS_CALC_DISTINCT, "true");
assertEquals(new EnumFieldValue(4, "Critical"),
assertEquals(new EnumFieldValue(11, "Critical"),
rsp.getFieldStatsInfo().get(fieldName).getMax());
handle.put("severity", UNORDERED); // this is stupid, but stats.facet doesn't garuntee order

433
solr/core/src/test/org/apache/solr/schema/EnumFieldTest.java
View File

@ -16,29 +16,61 @@
*/
package org.apache.solr.schema;
import java.util.Iterator;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.search.SolrQueryParser;
import org.junit.BeforeClass;
import org.junit.Test;
public class EnumFieldTest extends SolrTestCaseJ4 {
private final String FIELD_NAME = random().nextBoolean() ? "severity" : "severity_dv";
private final String FIELD_NAME = "severity";
private final String MV_FIELD_NAME = "severity_mv";
@BeforeClass
public static void beforeClass() throws Exception {
System.setProperty("solr.tests.EnumFieldTest.indexed", Boolean.toString(random().nextBoolean()));
doInitCore();
// System.out.println("solr.tests.numeric.dv: " + System.getProperty("solr.tests.numeric.dv"));
// System.out.println("solr.tests.EnumFieldTest.indexed: " + System.getProperty("solr.tests.EnumFieldTest.indexed"));
// System.out.println("solr.tests.EnumFieldType: " + System.getProperty("solr.tests.EnumFieldType"));
}
private static void doInitCore() throws Exception {
initCore("solrconfig-minimal.xml", "schema-enums.xml");
}
@Test
public void testEnumSchema() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
IndexSchema schema = h.getCore().getLatestSchema();
SchemaField enumField = schema.getField(FIELD_NAME);
assertNotNull(enumField);
SchemaField mvEnumField = schema.getField(MV_FIELD_NAME);
assertNotNull(mvEnumField);
}
@Test
public void testEnumRangeSearch() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
assumeFalse("Skipping testing of unindexed EnumField without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumField")
&& System.getProperty("solr.tests.EnumFieldTest.indexed").equals("false")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
clearIndex();
assertU(adoc("id", "0", FIELD_NAME, "Not Available"));
@ -65,62 +97,123 @@ public class EnumFieldTest extends SolrTestCaseJ4 {
assertU(commit());
//range with the same value
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":[\"Not Available\" TO \"Not Available\"]"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":[\"Not Available\" TO \"Not Available\"]"),
"//*[@numFound='5']");
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":[\"Not Available\" TO Critical]"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":[\"Not Available\" TO Critical]"),
"//*[@numFound='15']");
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":[Low TO High]"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":[Low TO High]"),
"//*[@numFound='9']");
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":[High TO Low]"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":[High TO Low]"),
"//*[@numFound='0']");
//with int values
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":[High TO 4]"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":[High TO 11]"),
"//*[@numFound='3']");
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":[3 TO Critical]"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":[3 TO Critical]"),
"//*[@numFound='3']");
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":[3 TO 4]"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":[3 TO 11]"),
"//*[@numFound='3']");
//exclusive
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":{Low TO High]"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":{Low TO High]"),
"//*[@numFound='5']");
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":[Low TO High}"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":[Low TO High}"),
"//*[@numFound='7']");
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":{Low TO High}"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":{Low TO High}"),
"//*[@numFound='3']");
//all docs
assertQ(req("fl", "" + FIELD_NAME, "q",
"*:*"),
assertQ(req("fl", "" + FIELD_NAME, "q", "*:*"),
"//*[@numFound='17']");
//all docs with values
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":[* TO *]"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":[* TO *]"),
"//*[@numFound='15']");
//empty docs
assertQ(req("fl", "" + FIELD_NAME, "q",
"-" + FIELD_NAME + ":[* TO *]"),
assertQ(req("fl", "" + FIELD_NAME, "q", "-" + FIELD_NAME + ":[* TO *]"),
"//*[@numFound='2']");
}
@Test
public void testMultivaluedEnumRangeSearch() {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
assumeFalse("Skipping testing of range searching over multivalued EnumField - see SOLR-11193",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumField"));
clearIndex();
assertU(adoc("id", "0", MV_FIELD_NAME, "Not Available")); // Single value
assertU(adoc("id", "1", MV_FIELD_NAME, "Not Available", MV_FIELD_NAME, "Critical"));
assertU(adoc("id", "2", MV_FIELD_NAME, "Not Available", MV_FIELD_NAME, "Critical"));
assertU(adoc("id", "3")); // No values
assertU(adoc("id", "4")); // No values
assertU(adoc("id", "5", MV_FIELD_NAME, "Low", MV_FIELD_NAME, "Medium"));
assertU(adoc("id", "6", MV_FIELD_NAME, "Low", MV_FIELD_NAME, "Medium"));
assertU(adoc("id", "7", MV_FIELD_NAME, "Low", MV_FIELD_NAME, "Medium"));
assertU(adoc("id", "8", MV_FIELD_NAME, "Low", MV_FIELD_NAME, "Critical"));
assertU(adoc("id", "9", MV_FIELD_NAME, "Medium")); // Single value
assertU(adoc("id", "10", MV_FIELD_NAME, "Medium", MV_FIELD_NAME, "High"));
assertU(adoc("id", "11", MV_FIELD_NAME, "Medium", MV_FIELD_NAME, "High"));
assertU(adoc("id", "12", MV_FIELD_NAME, "High", MV_FIELD_NAME, "Critical"));
assertU(adoc("id", "13", MV_FIELD_NAME, "High", MV_FIELD_NAME, "High")); // Two of same value
assertU(adoc("id", "14", MV_FIELD_NAME, "Critical", MV_FIELD_NAME, "Medium", MV_FIELD_NAME, "Not Available"));
assertU(commit());
//range with the same value
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":[\"Not Available\" TO \"Not Available\"]"),
"//*[@numFound='4']");
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":[\"Not Available\" TO Critical]"),
"//*[@numFound='13']");
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":[Low TO High]"),
"//*[@numFound='10']");
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":[High TO Low]"),
"//*[@numFound='0']");
//with int values
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":[High TO 11]"),
"//*[@numFound='8']");
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":[3 TO Critical]"),
"//*[@numFound='8']");
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":[3 TO 11]"),
"//*[@numFound='8']");
//exclusive
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":{Low TO High]"),
"//*[@numFound='9']");
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":[Low TO High}"),
"//*[@numFound='8']");
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":{Low TO High}"),
"//*[@numFound='7']");
//all docs
assertQ(req("fl", "" + MV_FIELD_NAME, "q", "*:*"),
"//*[@numFound='15']");
//all docs with values
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":[* TO *]"),
"//*[@numFound='13']");
//empty docs
assertQ(req("fl", "" + MV_FIELD_NAME, "q", "-" + MV_FIELD_NAME + ":[* TO *]"),
"//*[@numFound='2']");
}
@Test
public void testBogusEnumSearch() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
clearIndex();
assertU(adoc("id", "0", FIELD_NAME, "Not Available"));
@ -136,49 +229,130 @@ public class EnumFieldTest extends SolrTestCaseJ4 {
assertU(commit());
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":bla"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":bla"),
"//*[@numFound='0']");
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":7"),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":7"),
"//*[@numFound='0']");
assertQ(req("fl", "" + FIELD_NAME, "q",
FIELD_NAME + ":\"-3\""),
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":\"-3\""),
"//*[@numFound='0']");
}
@Test
public void testMultivaluedBogusEnumSearch() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
clearIndex();
assertU(adoc("id", "0", MV_FIELD_NAME, "Not Available", MV_FIELD_NAME, "Critical"));
assertU(adoc("id", "1", MV_FIELD_NAME, "Low", MV_FIELD_NAME, "Not Available"));
assertU(adoc("id", "2", MV_FIELD_NAME, "Medium", MV_FIELD_NAME, "Low"));
assertU(adoc("id", "3", MV_FIELD_NAME, "High", MV_FIELD_NAME, "Medium"));
assertU(adoc("id", "4", MV_FIELD_NAME, "Critical", MV_FIELD_NAME, "High"));
// two docs w/o values
for (int i = 8; i <= 9; i++) {
assertU(adoc("id", "" + i));
}
assertU(commit());
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":bla"),
"//*[@numFound='0']");
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":7"),
"//*[@numFound='0']");
assertQ(req("fl", "" + MV_FIELD_NAME, "q", MV_FIELD_NAME + ":\"-3\""),
"//*[@numFound='0']");
}
@Test
public void testBogusEnumIndexing() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
ignoreException("Unknown value for enum field: " + FIELD_NAME + ", value: blabla");
ignoreException("Unknown value for enum field: " + FIELD_NAME + ", value: 10");
ignoreException("Unknown value for enum field: " + FIELD_NAME + ", value: 145");
ignoreException("Unknown value for enum field: " + FIELD_NAME + ", value: -4");
clearIndex();
assertFailedU(adoc("id", "0", FIELD_NAME, "blabla"));
assertFailedU(adoc("id", "0", FIELD_NAME, "10"));
assertFailedU(adoc("id", "0", FIELD_NAME, "145"));
assertFailedU(adoc("id", "0", FIELD_NAME, "-4"));
}
@Test
public void testMultivaluedBogusEnumIndexing() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
ignoreException("Unknown value for enum field: " + MV_FIELD_NAME + ", value: blabla");
ignoreException("Unknown value for enum field: " + MV_FIELD_NAME + ", value: 145");
ignoreException("Unknown value for enum field: " + MV_FIELD_NAME + ", value: -4");
clearIndex();
assertFailedU(adoc("id", "0", MV_FIELD_NAME, "blabla", MV_FIELD_NAME, "High"));
assertFailedU(adoc("id", "0", MV_FIELD_NAME, "145", MV_FIELD_NAME, "Low"));
assertFailedU(adoc("id", "0", MV_FIELD_NAME, "-4", MV_FIELD_NAME, "Critical"));
}
@Test
public void testKnownIntegerEnumIndexing() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
clearIndex();
assertU(adoc("id", "0", FIELD_NAME, "1"));
assertU(adoc("id", "1", FIELD_NAME, "11"));
assertU(commit());
assertQ(req("fl", "" + FIELD_NAME, "q", "*:*"), "//doc[1]/str[@name='" + FIELD_NAME + "']/text()='Low'");
assertQ(req("fl", "id," + FIELD_NAME, "q", "*:*"),
"//doc[str[@name='id']/text()='0']/str[@name='" + FIELD_NAME + "']/text()='Low'",
"//doc[str[@name='id']/text()='1']/str[@name='" + FIELD_NAME + "']/text()='Critical'");
}
@Test
public void testMultivaluedKnownIntegerEnumIndexing() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
clearIndex();
assertU(adoc("id", "0", MV_FIELD_NAME, "1", MV_FIELD_NAME, "2"));
assertU(adoc("id", "1", MV_FIELD_NAME, "11", MV_FIELD_NAME, "3"));
assertU(commit());
assertQ(req("fl", "id," + MV_FIELD_NAME, "q", "*:*"),
"//doc[str[@name='id']/text()='0']/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Low'",
"//doc[str[@name='id']/text()='0']/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Medium'",
"//doc[str[@name='id']/text()='1']/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Critical'",
"//doc[str[@name='id']/text()='1']/arr[@name='" + MV_FIELD_NAME + "']/str/text()='High'");
}
@Test
public void testEnumSort() throws Exception {
clearIndex();
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
assumeFalse("Skipping testing of unindexed EnumField without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumField")
&& System.getProperty("solr.tests.EnumFieldTest.indexed").equals("false")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
clearIndex();
assertU(adoc("id", "0", FIELD_NAME, "Not Available"));
assertU(adoc("id", "1", FIELD_NAME, "Low"));
assertU(adoc("id", "2", FIELD_NAME, "Medium"));
@ -192,19 +366,192 @@ public class EnumFieldTest extends SolrTestCaseJ4 {
assertU(commit());
assertQ(req("fl", "" + FIELD_NAME, "q", "*:*", "sort", FIELD_NAME + " desc"), "//doc[1]/str[@name='" + FIELD_NAME + "']/text()='Critical'",
"//doc[2]/str[@name='" + FIELD_NAME + "']/text()='High'", "//doc[3]/str[@name='" + FIELD_NAME + "']/text()='Medium'", "//doc[4]/str[@name='" + FIELD_NAME + "']/text()='Low'",
assertQ(req("fl", "id," + FIELD_NAME, "q", "*:*", "sort", FIELD_NAME + " desc", "indent","on"),
"//doc[1]/str[@name='" + FIELD_NAME + "']/text()='Critical'",
"//doc[2]/str[@name='" + FIELD_NAME + "']/text()='High'",
"//doc[3]/str[@name='" + FIELD_NAME + "']/text()='Medium'",
"//doc[4]/str[@name='" + FIELD_NAME + "']/text()='Low'",
"//doc[5]/str[@name='" + FIELD_NAME + "']/text()='Not Available'");
//sort ascending - empty values will be first
assertQ(req("fl", "" + FIELD_NAME, "q", "*:*", "sort", FIELD_NAME + " asc"), "//doc[3]/str[@name='" + FIELD_NAME + "']/text()='Not Available'");
assertQ(req("fl", "id," + FIELD_NAME, "q", "*:*", "sort", FIELD_NAME + " asc", "indent", "on"),
"//doc[3]/str[@name='" + FIELD_NAME + "']/text()='Not Available'");
//q for not empty docs
assertQ(req("fl", "" + FIELD_NAME, "q", FIELD_NAME + ":[* TO *]" , "sort", FIELD_NAME + " asc"), "//doc[1]/str[@name='" + FIELD_NAME + "']/text()='Not Available'",
"//doc[2]/str[@name='" + FIELD_NAME + "']/text()='Low'", "//doc[3]/str[@name='" + FIELD_NAME + "']/text()='Medium'", "//doc[4]/str[@name='" + FIELD_NAME + "']/text()='High'",
assertQ(req("fl", "id," + FIELD_NAME, "q", FIELD_NAME + ":[* TO *]" , "sort", FIELD_NAME + " asc", "indent", "on"),
"//doc[1]/str[@name='" + FIELD_NAME + "']/text()='Not Available'",
"//doc[2]/str[@name='" + FIELD_NAME + "']/text()='Low'",
"//doc[3]/str[@name='" + FIELD_NAME + "']/text()='Medium'",
"//doc[4]/str[@name='" + FIELD_NAME + "']/text()='High'",
"//doc[5]/str[@name='" + FIELD_NAME + "']/text()='Critical'"
);
}
}
@Test
public void testMultivaluedEnumSort() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
assumeFalse("Skipping testing of sorting over multivalued EnumField - see SOLR-11193",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumField"));
clearIndex();
assertU(adoc("id", "0", MV_FIELD_NAME, "Not Available")); // Single value
assertU(adoc("id", "1", MV_FIELD_NAME, "Not Available", MV_FIELD_NAME, "Critical"));
assertU(adoc("id", "2", MV_FIELD_NAME, "Not Available", MV_FIELD_NAME, "Critical"));
assertU(adoc("id", "3")); // No values
assertU(adoc("id", "4")); // No values
assertU(adoc("id", "5", MV_FIELD_NAME, "Low", MV_FIELD_NAME, "Medium"));
assertU(adoc("id", "6", MV_FIELD_NAME, "Low", MV_FIELD_NAME, "Medium"));
assertU(adoc("id", "7", MV_FIELD_NAME, "Low", MV_FIELD_NAME, "Medium"));
assertU(adoc("id", "8", MV_FIELD_NAME, "Low", MV_FIELD_NAME, "Critical"));
assertU(adoc("id", "9", MV_FIELD_NAME, "Medium")); // Single value
assertU(adoc("id", "10", MV_FIELD_NAME, "Medium", MV_FIELD_NAME, "High"));
assertU(adoc("id", "11", MV_FIELD_NAME, "Medium", MV_FIELD_NAME, "High"));
assertU(adoc("id", "12", MV_FIELD_NAME, "High", MV_FIELD_NAME, "Critical"));
assertU(adoc("id", "13", MV_FIELD_NAME, "High", MV_FIELD_NAME, "High")); // Two of same value
assertU(adoc("id", "14", MV_FIELD_NAME, "Critical", MV_FIELD_NAME, "Medium", MV_FIELD_NAME, "Not Available"));
assertU(commit());
assertQ(req("fl", "id," + MV_FIELD_NAME, "q", "*:*", "sort", "field(" + MV_FIELD_NAME + ",min) desc", "indent","on"),
"//doc[1]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Critical'",
"//doc[2]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='High'",
"//doc[3]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Medium'",
"//doc[6]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Low'",
"//doc[10]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Not Available'");
//sort ascending - empty values will be first
assertQ(req("fl", "id," + MV_FIELD_NAME, "q", "*:*", "sort", "field(" + MV_FIELD_NAME + ",min) asc", "indent", "on"),
"//doc[3]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Not Available'");
//q for not empty docs
assertQ(req("fl", "id," + MV_FIELD_NAME,
"q", MV_FIELD_NAME + ":[* TO *]" , "sort", "field(" + MV_FIELD_NAME + ",'min') asc",
"rows","15", "indent","on"),
"//doc[1]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Not Available'",
"//doc[5]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Low'",
"//doc[9]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Medium'",
"//doc[10]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='High'",
"//doc[12]/arr[@name='" + MV_FIELD_NAME + "']/str/text()='Critical'");
}
@Test
public void testSetQuery() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
assumeFalse("Skipping testing of unindexed EnumField without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumField")
&& System.getProperty("solr.tests.EnumFieldTest.indexed").equals("false")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
clearIndex();
SchemaField sf = h.getCore().getLatestSchema().getField(FIELD_NAME);
Set<String> enumStrs = ((AbstractEnumField)sf.getType()).getEnumMapping().enumStringToIntMap.keySet();
assertTrue(enumStrs.size() > SolrQueryParser.TERMS_QUERY_THRESHOLD);
Iterator<String> enumStrIter = enumStrs.iterator();
for (int i = 0 ; enumStrIter.hasNext() ; ++i) {
assertU(adoc("id", "" + i, FIELD_NAME, enumStrIter.next()));
}
assertU(commit());
StringBuilder builder = new StringBuilder(FIELD_NAME + ":(");
enumStrs.forEach(v -> builder.append(v.replace(" ", "\\ ")).append(' '));
builder.append(')');
if (sf.indexed()) { // SolrQueryParser should also be generating a TermInSetQuery if indexed
String setQuery = sf.getType().getSetQuery(null, sf, enumStrs).toString();
if (sf.getType() instanceof EnumField) { // Trie field TermInSetQuery non-XML chars serialize with "#XX;" syntax
Pattern nonXMLCharPattern = Pattern.compile("[\u0000-\u0008\u000B\u000C\u000E-\u0019]");
StringBuffer munged = new StringBuffer();
Matcher matcher = nonXMLCharPattern.matcher(setQuery);
while (matcher.find()) {
matcher.appendReplacement(munged, "#" + (int)matcher.group(0).charAt(0) + ";");
}
matcher.appendTail(munged);
setQuery = munged.toString();
}
assertQ(req(CommonParams.DEBUG, CommonParams.QUERY, "q", "*:*", "fq", builder.toString(),
"fl", "id," + FIELD_NAME, "rows", "" + enumStrs.size(), "indent", "on"),
"//*[@numFound='" + enumStrs.size() + "']",
"//*[@name='parsed_filter_queries']/str[normalize-space(.)=normalize-space('TermInSetQuery(" + setQuery + ")')]"); // fix [\r\n] problems
} else {
// Won't use TermInSetQuery if the field is not indexed, but should match the same docs
assertQ(req(CommonParams.DEBUG, CommonParams.QUERY, "q", "*:*", "fq", builder.toString(), "fl", "id," + FIELD_NAME, "indent", "on"),
"//*[@numFound='" + enumStrs.size() + "']");
}
}
@Test
public void testMultivaluedSetQuery() throws Exception {
assumeFalse("Skipping testing of EnumFieldType without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
assumeFalse("Skipping testing of unindexed EnumField without docValues, which is unsupported.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumField")
&& System.getProperty("solr.tests.EnumFieldTest.indexed").equals("false")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
clearIndex();
SchemaField sf = h.getCore().getLatestSchema().getField(MV_FIELD_NAME);
Set<String> enumStrs = ((AbstractEnumField)sf.getType()).getEnumMapping().enumStringToIntMap.keySet();
assertTrue(enumStrs.size() > SolrQueryParser.TERMS_QUERY_THRESHOLD);
Iterator<String> enumStrIter = enumStrs.iterator();
String prevEnumStr = "x18"; // wrap around
for (int i = 0 ; enumStrIter.hasNext() ; ++i) {
String thisEnumStr = enumStrIter.next();
assertU(adoc("id", "" + i, MV_FIELD_NAME, thisEnumStr, MV_FIELD_NAME, prevEnumStr));
prevEnumStr = thisEnumStr;
}
assertU(commit());
StringBuilder builder = new StringBuilder(MV_FIELD_NAME + ":(");
enumStrs.forEach(v -> builder.append(v.replace(" ", "\\ ")).append(' '));
builder.append(')');
if (sf.indexed()) { // SolrQueryParser should also be generating a TermInSetQuery if indexed
String setQuery = sf.getType().getSetQuery(null, sf, enumStrs).toString();
if (sf.getType() instanceof EnumField) { // Trie field TermInSetQuery non-XML chars serialize with "#XX;" syntax
Pattern nonXMLCharPattern = Pattern.compile("[\u0000-\u0008\u000B\u000C\u000E-\u0019]");
StringBuffer munged = new StringBuffer();
Matcher matcher = nonXMLCharPattern.matcher(setQuery);
while (matcher.find()) {
matcher.appendReplacement(munged, "#" + (int)matcher.group(0).charAt(0) + ";");
}
matcher.appendTail(munged);
setQuery = munged.toString();
}
assertQ(req(CommonParams.DEBUG, CommonParams.QUERY, "q", "*:*", "fq", builder.toString(),
"fl", "id," + MV_FIELD_NAME, "rows", "" + enumStrs.size(), "indent", "on"),
"//*[@numFound='" + enumStrs.size() + "']",
"//*[@name='parsed_filter_queries']/str[normalize-space(.)=normalize-space('TermInSetQuery(" + setQuery + ")')]"); // fix [\r\n] problems
} else {
// Won't use TermInSetQuery if the field is not indexed, but should match the same docs
assertQ(req(CommonParams.DEBUG, CommonParams.QUERY, "q", "*:*", "fq", builder.toString(), "fl", "id," + MV_FIELD_NAME, "indent", "on"),
"//*[@numFound='" + enumStrs.size() + "']");
}
}
@Test
public void testEnumFieldTypeWithoutDocValues() throws Exception {
assumeTrue("Only testing EnumFieldType without docValues.",
System.getProperty("solr.tests.EnumFieldType").equals("solr.EnumFieldType")
&& System.getProperty("solr.tests.numeric.dv").equals("false"));
try {
deleteCore();
initCore("solrconfig-minimal.xml", "bad-schema-enums.xml");
SolrException e = expectThrows(SolrException.class,
() -> assertU(adoc("id", "0", FIELD_NAME, "Not Available")));
assertTrue(e.getMessage().contains("EnumFieldType requires docValues=\"true\""));
} finally { // put back the core expected by other tests
deleteCore();
doInitCore();
}
}
}

2
solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
View File

@ -336,7 +336,7 @@ public class TestUseDocValuesAsStored extends AbstractBadConfigTestBase {
// so cardinality depends on the value source
final int expectedCardinality =
(isStoredField(field) || (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)
&& ! (field.startsWith("enum") || field.startsWith("test_s"))))
&& ! field.startsWith("test_s")))
? value.length : valueSet.size();
xpaths[value.length] = "*[count(//arr[@name='"+field+"']/"+type+")="+expectedCardinality+"]";
assertU(adoc(fieldAndValues));

2
solr/solr-ref-guide/src/docvalues.adoc
View File

@ -47,7 +47,7 @@ DocValues are only available for specific field types. The types chosen determin
* `StrField` and `UUIDField`.
** If the field is single-valued (i.e., multi-valued is false), Lucene will use the SORTED type.
** If the field is multi-valued, Lucene will use the SORTED_SET type.
* Any `Trie*` numeric fields, date fields and `EnumField`.
* Any `Trie*` numeric fields, date fields and `EnumFieldType`.
** If the field is single-valued (i.e., multi-valued is false), Lucene will use the NUMERIC type.
** If the field is multi-valued, Lucene will use the SORTED_SET type.
* Boolean fields

3
solr/solr-ref-guide/src/field-types-included-with-solr.adoc
View File

@ -32,7 +32,8 @@ The following table lists the field types that are available in Solr. The `org.a
|CurrencyFieldType |Supports currencies and exchange rates. See the section <<working-with-currencies-and-exchange-rates.adoc#working-with-currencies-and-exchange-rates,Working with Currencies and Exchange Rates>>.
|DateRangeField |Supports indexing date ranges, to include point in time date instances as well (single-millisecond durations). See the section <<working-with-dates.adoc#working-with-dates,Working with Dates>> for more detail on using this field type. Consider using this field type even if it's just for date instances, particularly when the queries typically fall on UTC year/month/day/hour, etc., boundaries.
|ExternalFileField |Pulls values from a file on disk. See the section <<working-with-external-files-and-processes.adoc#working-with-external-files-and-processes,Working with External Files and Processes>>.
|EnumField |Allows defining an enumerated set of values which may not be easily sorted by either alphabetic or numeric order (such as a list of severities, for example). This field type takes a configuration file, which lists the proper order of the field values. See the section <<working-with-enum-fields.adoc#working-with-enum-fields,Working with Enum Fields>> for more information.
|EnumField |Deprecated in favor of EnumFieldType
|EnumFieldType |Allows defining an enumerated set of values which may not be easily sorted by either alphabetic or numeric order (such as a list of severities, for example). This field type takes a configuration file, which lists the proper order of the field values. See the section <<working-with-enum-fields.adoc#working-with-enum-fields,Working with Enum Fields>> for more information.
|ICUCollationField |Supports Unicode collation for sorting and range queries. See the section <<language-analysis.adoc#unicode-collation,Unicode Collation>>.
|LatLonPointSpatialField |<<spatial-search.adoc#spatial-search,Spatial Search>>: a latitude/longitude coordinate pair; possibly multi-valued for multiple points. Usually it's specified as "lat,lon" order with a comma.
|LatLonType |(deprecated) <<spatial-search.adoc#spatial-search,Spatial Search>>: a single-valued latitude/longitude coordinate pair. Usually it's specified as "lat,lon" order with a comma.

20
solr/solr-ref-guide/src/working-with-enum-fields.adoc
View File

@ -18,16 +18,22 @@
// specific language governing permissions and limitations
// under the License.
The EnumField type allows defining a field whose values are a closed set, and the sort order is pre-determined but is not alphabetic nor numeric. Examples of this are severity lists, or risk definitions.
EnumFieldType allows defining a field whose values are a closed set, and the sort order is pre-determined but is not alphabetic nor numeric. Examples of this are severity lists, or risk definitions.
== Defining an EnumField in schema.xml
.EnumField has been Deprecated
[WARNING]
====
EnumField has been deprecated in favor of EnumFieldType; all configuration examples below use EnumFieldType.
====
The EnumField type definition is quite simple, as in this example defining field types for "priorityLevel" and "riskLevel" enumerations:
== Defining an EnumFieldType in schema.xml
The EnumFieldType type definition is quite simple, as in this example defining field types for "priorityLevel" and "riskLevel" enumerations:
[source,xml]
----
<fieldType name="priorityLevel" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="priority"/>
<fieldType name="riskLevel" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="risk" />
<fieldType name="priorityLevel" class="solr.EnumFieldType" docValues="true" enumsConfig="enumsConfig.xml" enumName="priority"/>
<fieldType name="riskLevel" class="solr.EnumFieldType" docValues="true" enumsConfig="enumsConfig.xml" enumName="risk" />
----
Besides the `name` and the `class`, which are common to all field types, this type also takes two additional parameters:
@ -35,7 +41,9 @@ Besides the `name` and the `class`, which are common to all field types, this ty
`enumsConfig`:: the name of a configuration file that contains the `<enum/>` list of field values and their order that you wish to use with this field type. If a path to the file is not defined specified, the file should be in the `conf` directory for the collection.
`enumName`:: the name of the specific enumeration in the `enumsConfig` file to use for this type.
== Defining the EnumField Configuration File
Note that `docValues="true"` must be specified either in the EnumFieldType fieldType or field specification.
== Defining the EnumFieldType Configuration File
The file named with the `enumsConfig` parameter can contain multiple enumeration value lists with different names if there are multiple uses for enumerations in your Solr schema.

2
solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
View File

@ -2720,6 +2720,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
private_RANDOMIZED_NUMERIC_FIELDTYPES.put(Long.class, "solr.TrieLongField");
private_RANDOMIZED_NUMERIC_FIELDTYPES.put(Double.class, "solr.TrieDoubleField");
private_RANDOMIZED_NUMERIC_FIELDTYPES.put(Date.class, "solr.TrieDateField");
private_RANDOMIZED_NUMERIC_FIELDTYPES.put(Enum.class, "solr.EnumField");
System.setProperty(NUMERIC_POINTS_SYSPROP, "false");
} else {
@ -2731,6 +2732,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
private_RANDOMIZED_NUMERIC_FIELDTYPES.put(Long.class, "solr.LongPointField");
private_RANDOMIZED_NUMERIC_FIELDTYPES.put(Double.class, "solr.DoublePointField");
private_RANDOMIZED_NUMERIC_FIELDTYPES.put(Date.class, "solr.DatePointField");
private_RANDOMIZED_NUMERIC_FIELDTYPES.put(Enum.class, "solr.EnumFieldType");
System.setProperty(NUMERIC_POINTS_SYSPROP, "true");
}