mirror of https://github.com/apache/lucene.git
SOLR-14590 : Add support for Lucene's FeatureField in Solr (#1620)
Add a new RankField type that internally creates a FeatureField Add a new RankQParser that can create queries on the FeatureField
This commit is contained in:
Tomas Fernandez Lobbe
GitHub
parent
d1c29ae8a9
commit
6eb7bc3b7b
|
|
@ -119,6 +119,10 @@ New Features
|
|||
|
||||
* SOLR-14599: Package manager support for cluster level plugins (see SOLR-14404) (Ishan Chattopadhyaya)
|
||||
|
||||
* SOLR-14590: Add support for RankFields. RankFields allow the use of per-document scoring factors in a
|
||||
way that lets Solr skip over non-competitive documents when ranking. See SOLR-13289.
|
||||
(Tomás Fernández Löbbe, Varun Thacker)
|
||||
|
||||
Improvements
|
||||
---------------------
|
||||
* SOLR-14316: Remove unchecked type conversion warning in JavaBinCodec's readMapEntry's equals() method
|
||||
|
|
|
|||
|
|
@ -0,0 +1,140 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.schema;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.lucene.document.FeatureField;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
import org.apache.lucene.index.IndexableFieldType;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.response.TextResponseWriter;
|
||||
import org.apache.solr.search.QParser;
|
||||
import org.apache.solr.search.RankQParserPlugin;
|
||||
import org.apache.solr.uninverting.UninvertingReader.Type;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* {@code RankField}s can be used to store scoring factors to improve document ranking. They should be used
|
||||
* in combination with {@link RankQParserPlugin}. To use:
|
||||
* </p>
|
||||
* <p>
|
||||
* Define the {@code RankField} {@code fieldType} in your schema:
|
||||
* </p>
|
||||
* <pre class="prettyprint">
|
||||
* <fieldType name="rank" class="solr.RankField" />
|
||||
* </pre>
|
||||
* <p>
|
||||
* Add fields to the schema, i.e.:
|
||||
* </p>
|
||||
* <pre class="prettyprint">
|
||||
* <field name="pagerank" type="rank" />
|
||||
* </pre>
|
||||
*
|
||||
* Query using the {@link RankQParserPlugin}, for example
|
||||
* <pre class="prettyprint">
|
||||
* http://localhost:8983/solr/techproducts?q=memory _query_:{!rank f='pagerank', function='log' scalingFactor='1.2'}
|
||||
* </pre>
|
||||
*
|
||||
* @see RankQParserPlugin
|
||||
* @lucene.experimental
|
||||
* @since 8.6
|
||||
*/
|
||||
public class RankField extends FieldType {
|
||||
|
||||
/*
|
||||
* While the user can create multiple RankFields, internally we use a single Lucene field,
|
||||
* and we map the Solr field name to the "feature" in Lucene's FeatureField. This is mainly
|
||||
* to simplify the user experience.
|
||||
*/
|
||||
public static final String INTERNAL_RANK_FIELD_NAME = "_rank_";
|
||||
|
||||
@Override
|
||||
public Type getUninversionType(SchemaField sf) {
|
||||
throw null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void init(IndexSchema schema, Map<String,String> args) {
|
||||
super.init(schema, args);
|
||||
if (schema.getFieldOrNull(INTERNAL_RANK_FIELD_NAME) != null) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A field named \"" + INTERNAL_RANK_FIELD_NAME + "\" can't be defined in the schema");
|
||||
}
|
||||
for (int prop:new int[] {STORED, DOC_VALUES, OMIT_TF_POSITIONS, SORT_MISSING_FIRST, SORT_MISSING_LAST}) {
|
||||
if ((trueProperties & prop) != 0) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Property \"" + getPropertyName(prop) + "\" can't be set to true in RankFields");
|
||||
}
|
||||
}
|
||||
for (int prop:new int[] {UNINVERTIBLE, INDEXED, MULTIVALUED}) {
|
||||
if ((falseProperties & prop) != 0) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Property \"" + getPropertyName(prop) + "\" can't be set to false in RankFields");
|
||||
}
|
||||
}
|
||||
properties &= ~(UNINVERTIBLE | STORED | DOC_VALUES);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
protected IndexableField createField(String name, String val, IndexableFieldType type) {
|
||||
if (val == null || val.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
float featureValue;
|
||||
try {
|
||||
featureValue = Float.parseFloat(val);
|
||||
} catch (NumberFormatException nfe) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error while creating field '" + name + "' from value '" + val + "'. Expecting float.", nfe);
|
||||
}
|
||||
// Internally, we always use the same field
|
||||
return new FeatureField(INTERNAL_RANK_FIELD_NAME, name, featureValue);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query getExistenceQuery(QParser parser, SchemaField field) {
|
||||
return new TermQuery(new Term(INTERNAL_RANK_FIELD_NAME, field.getName()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"Only a \"*\" term query can be done on RankFields");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Query getSpecializedRangeQuery(QParser parser, SchemaField field, String part1, String part2,
|
||||
boolean minInclusive, boolean maxInclusive) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"Range queries not supported on RankFields");
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortField getSortField(SchemaField field, boolean top) {
|
||||
// We could use FeatureField.newFeatureSort()
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"can not sort on a rank field: " + field.getName());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -87,6 +87,7 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI
|
|||
map.put(BoolQParserPlugin.NAME, new BoolQParserPlugin());
|
||||
map.put(MinHashQParserPlugin.NAME, new MinHashQParserPlugin());
|
||||
map.put(HashRangeQParserPlugin.NAME, new HashRangeQParserPlugin());
|
||||
map.put(RankQParserPlugin.NAME, new RankQParserPlugin());
|
||||
|
||||
standardPlugins = Collections.unmodifiableMap(map);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,158 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.search;
|
||||
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.lucene.document.FeatureField;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.schema.RankField;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
/**
|
||||
* {@code RankQParserPlugin} can be used to introduce document-depending scoring factors to ranking.
|
||||
* While this {@code QParser} delivers a (subset of) functionality already available via {@link FunctionQParser},
|
||||
* the benefit is that {@code RankQParserPlugin} can be used in combination with the {@code minExactCount} to
|
||||
* use BlockMax-WAND algorithm (skip non-competitive documents) to provide faster responses.
|
||||
*
|
||||
* @see RankField
|
||||
*
|
||||
* @lucene.experimental
|
||||
* @since 8.6
|
||||
*/
|
||||
public class RankQParserPlugin extends QParserPlugin {
|
||||
|
||||
public static final String NAME = "rank";
|
||||
public static final String FIELD = "f";
|
||||
public static final String FUNCTION = "function";
|
||||
public static final String WEIGHT = "weight";
|
||||
public static final String PIVOT = "pivot";
|
||||
public static final String SCALING_FACTOR = "scalingFactor";
|
||||
public static final String EXPONENT = "exponent";
|
||||
|
||||
private final static FeatureFieldFunction DEFAULT_FUNCTION = FeatureFieldFunction.SATU;
|
||||
|
||||
private enum FeatureFieldFunction {
|
||||
SATU {
|
||||
@Override
|
||||
public Query createQuery(String fieldName, SolrParams params) throws SyntaxError {
|
||||
Float weight = params.getFloat(WEIGHT);
|
||||
Float pivot = params.getFloat(PIVOT);
|
||||
if (pivot == null && (weight == null || Float.compare(weight.floatValue(), 1f) == 0)) {
|
||||
// No IAE expected in this case
|
||||
return FeatureField.newSaturationQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName);
|
||||
}
|
||||
if (pivot == null) {
|
||||
throw new SyntaxError("A pivot value needs to be provided if the weight is not 1 on \"satu\" function");
|
||||
}
|
||||
if (weight == null) {
|
||||
weight = Float.valueOf(1);
|
||||
}
|
||||
try {
|
||||
return FeatureField.newSaturationQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName, weight, pivot);
|
||||
} catch (IllegalArgumentException iae) {
|
||||
throw new SyntaxError(iae.getMessage());
|
||||
}
|
||||
}
|
||||
},
|
||||
LOG {
|
||||
@Override
|
||||
public Query createQuery(String fieldName, SolrParams params) throws SyntaxError {
|
||||
float weight = params.getFloat(WEIGHT, 1f);
|
||||
float scalingFactor = params.getFloat(SCALING_FACTOR, 1f);
|
||||
try {
|
||||
return FeatureField.newLogQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName, weight, scalingFactor);
|
||||
} catch (IllegalArgumentException iae) {
|
||||
throw new SyntaxError(iae.getMessage());
|
||||
}
|
||||
}
|
||||
},
|
||||
SIGM {
|
||||
@Override
|
||||
public Query createQuery(String fieldName, SolrParams params) throws SyntaxError {
|
||||
float weight = params.getFloat(WEIGHT, 1f);
|
||||
Float pivot = params.getFloat(PIVOT);
|
||||
if (pivot == null) {
|
||||
throw new SyntaxError("A pivot value needs to be provided when using \"sigm\" function");
|
||||
}
|
||||
Float exponent = params.getFloat(EXPONENT);
|
||||
if (exponent == null) {
|
||||
throw new SyntaxError("An exponent value needs to be provided when using \"sigm\" function");
|
||||
}
|
||||
try {
|
||||
return FeatureField.newSigmoidQuery(RankField.INTERNAL_RANK_FIELD_NAME, fieldName, weight, pivot, exponent);
|
||||
} catch (IllegalArgumentException iae) {
|
||||
throw new SyntaxError(iae.getMessage());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
public abstract Query createQuery(String fieldName, SolrParams params) throws SyntaxError;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
Objects.requireNonNull(localParams, "LocalParams String can't be null");
|
||||
Objects.requireNonNull(req, "SolrQueryRequest can't be null");
|
||||
return new RankQParser(qstr, localParams, params, req);
|
||||
}
|
||||
|
||||
public static class RankQParser extends QParser {
|
||||
|
||||
private final String field;
|
||||
|
||||
public RankQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
super(qstr, localParams, params, req);
|
||||
this.field = localParams.get(FIELD);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query parse() throws SyntaxError {
|
||||
if (this.field == null || this.field.isEmpty()) {
|
||||
throw new SyntaxError("Field can't be empty in rank queries");
|
||||
}
|
||||
SchemaField schemaField = req.getSchema().getFieldOrNull(field);
|
||||
if (schemaField == null) {
|
||||
throw new SyntaxError("Field \"" + this.field + "\" not found");
|
||||
}
|
||||
if (!(schemaField.getType() instanceof RankField)) {
|
||||
throw new SyntaxError("Field \"" + this.field + "\" is not a RankField");
|
||||
}
|
||||
return getFeatureFieldFunction(localParams.get(FUNCTION))
|
||||
.createQuery(field, localParams);
|
||||
}
|
||||
|
||||
private FeatureFieldFunction getFeatureFieldFunction(String function) throws SyntaxError {
|
||||
FeatureFieldFunction f = null;
|
||||
if (function == null || function.isEmpty()) {
|
||||
f = DEFAULT_FUNCTION;
|
||||
} else {
|
||||
try {
|
||||
f = FeatureFieldFunction.valueOf(function.toUpperCase(Locale.ROOT));
|
||||
} catch (IllegalArgumentException iae) {
|
||||
throw new SyntaxError("Unknown function in rank query: \"" + function + "\"");
|
||||
}
|
||||
}
|
||||
return f;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<schema name="rank_fields" version="1.6">
|
||||
<fieldType name="string" class="solr.StrField" />
|
||||
<fieldType name="rank" class="solr.RankField" />
|
||||
<fields>
|
||||
<field name="id" type="string" />
|
||||
<field name="str_field" type="string" />
|
||||
<field name="rank_1" type="rank" />
|
||||
<field name="rank_2" type="rank" />
|
||||
</fields>
|
||||
</schema>
|
||||
|
|
@ -41,6 +41,7 @@
|
|||
<fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
|
||||
<fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
|
||||
<fieldType name="currency" class="solr.CurrencyField" currencyConfig="currency.xml" multiValued="false"/>
|
||||
<fieldType name="rank" class="solr.RankField"/>
|
||||
|
||||
<!-- Field type demonstrating an Analyzer failure -->
|
||||
<fieldType name="failtype1" class="solr.TextField">
|
||||
|
|
@ -614,6 +615,8 @@
|
|||
<dynamicField name="ignored_*" type="ignored" multiValued="true"/>
|
||||
<dynamicField name="attr_*" type="text" indexed="true" stored="true" multiValued="true"/>
|
||||
|
||||
<dynamicField name="rank_*" type="rank"/>
|
||||
|
||||
<dynamicField name="random_*" type="random"/>
|
||||
|
||||
<dynamicField name="*_dpf" type="delimited_payloads_float" indexed="true" stored="true"/>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,285 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.schema;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import javax.xml.xpath.XPathConstants;
|
||||
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.util.TestHarness;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
|
||||
public class RankFieldTest extends SolrTestCaseJ4 {
|
||||
|
||||
private static final String RANK_1 = "rank_1";
|
||||
private static final String RANK_2 = "rank_2";
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
initCore("solrconfig-minimal.xml","schema-rank-fields.xml");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setUp() throws Exception {
|
||||
clearIndex();
|
||||
assertU(commit());
|
||||
super.setUp();
|
||||
}
|
||||
|
||||
public void testInternalFieldName() {
|
||||
assertEquals("RankField.INTERNAL_RANK_FIELD_NAME changed in an incompatible way",
|
||||
"_rank_", RankField.INTERNAL_RANK_FIELD_NAME);
|
||||
}
|
||||
|
||||
public void testBasic() {
|
||||
assertNotNull(h.getCore().getLatestSchema().getFieldOrNull(RANK_1));
|
||||
assertEquals(RankField.class, h.getCore().getLatestSchema().getField(RANK_1).getType().getClass());
|
||||
}
|
||||
|
||||
public void testBadFormat() {
|
||||
ignoreException("Expecting float");
|
||||
assertFailedU(adoc(
|
||||
"id", "1",
|
||||
RANK_1, "foo"
|
||||
));
|
||||
|
||||
assertFailedU(adoc(
|
||||
"id", "1",
|
||||
RANK_1, "1.2.3"
|
||||
));
|
||||
|
||||
unIgnoreException("Expecting float");
|
||||
|
||||
ignoreException("must be finite");
|
||||
assertFailedU(adoc(
|
||||
"id", "1",
|
||||
RANK_1, Float.toString(Float.POSITIVE_INFINITY)
|
||||
));
|
||||
|
||||
assertFailedU(adoc(
|
||||
"id", "1",
|
||||
RANK_1, Float.toString(Float.NEGATIVE_INFINITY)
|
||||
));
|
||||
|
||||
assertFailedU(adoc(
|
||||
"id", "1",
|
||||
RANK_1, Float.toString(Float.NaN)
|
||||
));
|
||||
|
||||
unIgnoreException("must be finite");
|
||||
|
||||
ignoreException("must be a positive");
|
||||
assertFailedU(adoc(
|
||||
"id", "1",
|
||||
RANK_1, Float.toString(-0.0f)
|
||||
));
|
||||
|
||||
assertFailedU(adoc(
|
||||
"id", "1",
|
||||
RANK_1, Float.toString(-1f)
|
||||
));
|
||||
|
||||
assertFailedU(adoc(
|
||||
"id", "1",
|
||||
RANK_1, Float.toString(0.0f)
|
||||
));
|
||||
unIgnoreException("must be a positive");
|
||||
}
|
||||
|
||||
public void testAddRandom() {
|
||||
for (int i = 0 ; i < random().nextInt(TEST_NIGHTLY ? 10000 : 100); i++) {
|
||||
assertU(adoc(
|
||||
"id", String.valueOf(i),
|
||||
RANK_1, Float.toString(random().nextFloat())
|
||||
));
|
||||
}
|
||||
assertU(commit());
|
||||
}
|
||||
|
||||
public void testSkipEmpty() {
|
||||
assertU(adoc(
|
||||
"id", "1",
|
||||
RANK_1, ""
|
||||
));
|
||||
}
|
||||
|
||||
public void testBasicAdd() throws IOException {
|
||||
assertU(adoc(
|
||||
"id", "testBasicAdd",
|
||||
RANK_1, "1"
|
||||
));
|
||||
assertU(commit());
|
||||
//assert that the document made it in
|
||||
assertQ(req("q", "id:testBasicAdd"), "//*[@numFound='1']");
|
||||
h.getCore().withSearcher((searcher) -> {
|
||||
LeafReader reader = searcher.getIndexReader().getContext().leaves().get(0).reader();
|
||||
// assert that the field made it in
|
||||
assertNotNull(reader.getFieldInfos().fieldInfo(RankField.INTERNAL_RANK_FIELD_NAME));
|
||||
// assert that the feature made it in
|
||||
assertTrue(reader.terms(RankField.INTERNAL_RANK_FIELD_NAME).iterator().seekExact(new BytesRef(RANK_1.getBytes(StandardCharsets.UTF_8))));
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
public void testMultipleRankFields() throws IOException {
|
||||
assertU(adoc(
|
||||
"id", "testMultiValueAdd",
|
||||
RANK_1, "1",
|
||||
RANK_2, "2"
|
||||
));
|
||||
assertU(commit());
|
||||
//assert that the document made it in
|
||||
assertQ(req("q", "id:testMultiValueAdd"), "//*[@numFound='1']");
|
||||
h.getCore().withSearcher((searcher) -> {
|
||||
LeafReader reader = searcher.getIndexReader().getContext().leaves().get(0).reader();
|
||||
// assert that the field made it in
|
||||
assertNotNull(reader.getFieldInfos().fieldInfo(RankField.INTERNAL_RANK_FIELD_NAME));
|
||||
// assert that the features made it in
|
||||
assertTrue(reader.terms(RankField.INTERNAL_RANK_FIELD_NAME).iterator().seekExact(new BytesRef(RANK_2.getBytes(StandardCharsets.UTF_8))));
|
||||
assertTrue(reader.terms(RankField.INTERNAL_RANK_FIELD_NAME).iterator().seekExact(new BytesRef(RANK_1.getBytes(StandardCharsets.UTF_8))));
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
public void testSortFails() throws IOException {
|
||||
assertU(adoc(
|
||||
"id", "testSortFails",
|
||||
RANK_1, "1"
|
||||
));
|
||||
assertU(commit());
|
||||
assertQEx("Can't sort on rank field", req(
|
||||
"q", "id:testSortFails",
|
||||
"sort", RANK_1 + " desc"), 400);
|
||||
}
|
||||
|
||||
@Ignore("We currently don't fail these kinds of requests with other field types")
|
||||
public void testFacetFails() throws IOException {
|
||||
assertU(adoc(
|
||||
"id", "testFacetFails",
|
||||
RANK_1, "1"
|
||||
));
|
||||
assertU(commit());
|
||||
assertQEx("Can't facet on rank field", req(
|
||||
"q", "id:testFacetFails",
|
||||
"facet", "true",
|
||||
"facet.field", RANK_1), 400);
|
||||
}
|
||||
|
||||
public void testTermQuery() throws IOException {
|
||||
assertU(adoc(
|
||||
"id", "testTermQuery",
|
||||
RANK_1, "1",
|
||||
RANK_2, "1"
|
||||
));
|
||||
assertU(adoc(
|
||||
"id", "testTermQuery2",
|
||||
RANK_1, "1"
|
||||
));
|
||||
assertU(commit());
|
||||
assertQ(req("q", RANK_1 + ":*"), "//*[@numFound='2']");
|
||||
assertQ(req("q", RANK_1 + ":[* TO *]"), "//*[@numFound='2']");
|
||||
assertQ(req("q", RANK_2 + ":*"), "//*[@numFound='1']");
|
||||
assertQ(req("q", RANK_2 + ":[* TO *]"), "//*[@numFound='1']");
|
||||
|
||||
assertQEx("Term queries not supported", req("q", RANK_1 + ":1"), 400);
|
||||
assertQEx("Range queries not supported", req("q", RANK_1 + ":[1 TO 10]"), 400);
|
||||
}
|
||||
|
||||
|
||||
public void testResponseQuery() throws IOException {
|
||||
assertU(adoc(
|
||||
"id", "testResponseQuery",
|
||||
RANK_1, "1"
|
||||
));
|
||||
assertU(commit());
|
||||
// Ignore requests to retrieve rank
|
||||
assertQ(req("q", RANK_1 + ":*",
|
||||
"fl", "id," + RANK_1),
|
||||
"//*[@numFound='1']",
|
||||
"count(//result/doc[1]/str)=1");
|
||||
}
|
||||
|
||||
public void testRankQParserQuery() throws IOException {
|
||||
assertU(adoc(
|
||||
"id", "1",
|
||||
"str_field", "foo",
|
||||
RANK_1, "1",
|
||||
RANK_2, "2"
|
||||
));
|
||||
assertU(adoc(
|
||||
"id", "2",
|
||||
"str_field", "foo",
|
||||
RANK_1, "2",
|
||||
RANK_2, "1"
|
||||
));
|
||||
assertU(commit());
|
||||
assertQ(req("q", "str_field:foo _query_:{!rank f='" + RANK_1 + "' function='log' scalingFactor='1'}"),
|
||||
"//*[@numFound='2']",
|
||||
"//result/doc[1]/str[@name='id'][.='2']",
|
||||
"//result/doc[2]/str[@name='id'][.='1']");
|
||||
|
||||
assertQ(req("q", "str_field:foo _query_:{!rank f='" + RANK_2 + "' function='log' scalingFactor='1'}"),
|
||||
"//*[@numFound='2']",
|
||||
"//result/doc[1]/str[@name='id'][.='1']",
|
||||
"//result/doc[2]/str[@name='id'][.='2']");
|
||||
|
||||
assertQ(req("q", "foo",
|
||||
"defType", "dismax",
|
||||
"qf", "str_field^10",
|
||||
"bq", "{!rank f='" + RANK_1 + "' function='log' scalingFactor='1'}"
|
||||
),
|
||||
"//*[@numFound='2']",
|
||||
"//result/doc[1]/str[@name='id'][.='2']",
|
||||
"//result/doc[2]/str[@name='id'][.='1']");
|
||||
|
||||
assertQ(req("q", "foo",
|
||||
"defType", "dismax",
|
||||
"qf", "str_field^10",
|
||||
"bq", "{!rank f='" + RANK_2 + "' function='log' scalingFactor='1'}"
|
||||
),
|
||||
"//*[@numFound='2']",
|
||||
"//result/doc[1]/str[@name='id'][.='1']",
|
||||
"//result/doc[2]/str[@name='id'][.='2']");
|
||||
}
|
||||
|
||||
public void testScoreChanges() throws Exception {
|
||||
assertU(adoc(
|
||||
"id", "1",
|
||||
"str_field", "foo",
|
||||
RANK_1, "1"
|
||||
));
|
||||
assertU(commit());
|
||||
ModifiableSolrParams params = params("q", "foo",
|
||||
"defType", "dismax",
|
||||
"qf", "str_field^10",
|
||||
"fl", "id,score",
|
||||
"wt", "xml");
|
||||
|
||||
double scoreBefore = (Double) TestHarness.evaluateXPath(h.query(req(params)), "//result/doc[1]/float[@name='score']", XPathConstants.NUMBER);
|
||||
params.add("bq", "{!rank f='" + RANK_1 + "' function='log' scalingFactor='1'}");
|
||||
double scoreAfter = (Double) TestHarness.evaluateXPath(h.query(req(params)), "//result/doc[1]/float[@name='score']", XPathConstants.NUMBER);
|
||||
assertNotEquals("Expecting score to change", scoreBefore, scoreAfter, 0f);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -356,6 +356,18 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
|
|||
req.close();
|
||||
}
|
||||
}
|
||||
|
||||
public void testRankQuery() throws Exception {
|
||||
SolrQueryRequest req = req("df", "foo_s");
|
||||
try {
|
||||
assertQueryEquals("rank", req,
|
||||
"{!rank f='rank_1'}",
|
||||
"{!rank f='rank_1' function='satu'}",
|
||||
"{!rank f='rank_1' function='satu' weight=1}");
|
||||
} finally {
|
||||
req.close();
|
||||
}
|
||||
}
|
||||
|
||||
public void testQueryNested() throws Exception {
|
||||
SolrQueryRequest req = req("df", "foo_s");
|
||||
|
|
|
|||
|
|
@ -0,0 +1,258 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.search;
|
||||
|
||||
import static org.apache.solr.search.RankQParserPlugin.EXPONENT;
|
||||
import static org.apache.solr.search.RankQParserPlugin.FIELD;
|
||||
import static org.apache.solr.search.RankQParserPlugin.FUNCTION;
|
||||
import static org.apache.solr.search.RankQParserPlugin.NAME;
|
||||
import static org.apache.solr.search.RankQParserPlugin.PIVOT;
|
||||
import static org.apache.solr.search.RankQParserPlugin.SCALING_FACTOR;
|
||||
import static org.apache.solr.search.RankQParserPlugin.WEIGHT;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.schema.RankField;
|
||||
import org.apache.solr.search.RankQParserPlugin.RankQParser;
|
||||
import org.hamcrest.CoreMatchers;
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
public class RankQParserPluginTest extends SolrTestCaseJ4 {
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
initCore("solrconfig-minimal.xml", "schema-rank-fields.xml");
|
||||
}
|
||||
|
||||
public void testParamCompatibility() {
|
||||
assertEquals("RankQParserPlugin.NAME changed in an incompatible way", "rank", NAME);
|
||||
assertEquals("RankQParserPlugin.FIELD changed in an incompatible way", "f", FIELD);
|
||||
assertEquals("RankQParserPlugin.FUNCTION changed in an incompatible way", "function", FUNCTION);
|
||||
assertEquals("RankQParserPlugin.PIVOT changed in an incompatible way", "pivot", PIVOT);
|
||||
assertEquals("RankQParserPlugin.SCALING_FACTOR changed in an incompatible way", "scalingFactor", SCALING_FACTOR);
|
||||
assertEquals("RankQParserPlugin.WEIGHT changed in an incompatible way", "weight", WEIGHT);
|
||||
assertEquals("RankQParserPlugin.EXPONENT changed in an incompatible way", "exponent", EXPONENT);
|
||||
}
|
||||
|
||||
public void testCreateParser() throws IOException {
|
||||
try (RankQParserPlugin rankQPPlugin = new RankQParserPlugin()) {
|
||||
QParser parser = rankQPPlugin.createParser("", new ModifiableSolrParams(), null, req());
|
||||
assertNotNull(parser);
|
||||
assertTrue(parser instanceof RankQParser);
|
||||
}
|
||||
}
|
||||
|
||||
public void testSyntaxErrors() throws IOException, SyntaxError {
|
||||
assertSyntaxError("No Field", "Field can't be empty", () ->
|
||||
getRankQParser(new ModifiableSolrParams(), null, req()).parse());
|
||||
assertSyntaxError("Field empty", "Field can't be empty", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, ""), null, req()).parse());
|
||||
assertSyntaxError("Field doesn't exist", "Field \"foo\" not found", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "foo"), null, req()).parse());
|
||||
assertSyntaxError("ID is not a feature field", "Field \"id\" is not a RankField", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "id"), null, req()).parse());
|
||||
}
|
||||
|
||||
public void testBadLogParameters() throws IOException, SyntaxError {
|
||||
assertSyntaxError("Expecting bad weight", "weight must be in", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "log",
|
||||
WEIGHT, "0"), null, req()).parse());
|
||||
assertSyntaxError("Expecting bad scaling factor", "scalingFactor must be", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "log",
|
||||
SCALING_FACTOR, "0"), null, req()).parse());
|
||||
}
|
||||
|
||||
public void testBadSaturationParameters() throws IOException, SyntaxError {
|
||||
assertSyntaxError("Expecting a pivot value", "A pivot value", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "satu",
|
||||
WEIGHT, "2"), null, req()).parse());
|
||||
assertSyntaxError("Expecting bad weight", "weight must be in", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "satu",
|
||||
PIVOT, "1",
|
||||
WEIGHT, "-1"), null, req()).parse());
|
||||
}
|
||||
|
||||
public void testBadSigmoidParameters() throws IOException, SyntaxError {
|
||||
assertSyntaxError("Expecting missing pivot", "A pivot value", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "sigm",
|
||||
EXPONENT, "1"), null, req()).parse());
|
||||
assertSyntaxError("Expecting missing exponent", "An exponent value", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "sigm",
|
||||
PIVOT, "1"), null, req()).parse());
|
||||
assertSyntaxError("Expecting bad weight", "weight must be in", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "sigm",
|
||||
PIVOT, "1",
|
||||
EXPONENT, "1",
|
||||
WEIGHT, "-1"), null, req()).parse());
|
||||
assertSyntaxError("Expecting bad pivot", "pivot must be", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "sigm",
|
||||
PIVOT, "0",
|
||||
EXPONENT, "1"), null, req()).parse());
|
||||
assertSyntaxError("Expecting bad exponent", "exp must be", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "sigm",
|
||||
PIVOT, "1",
|
||||
EXPONENT, "0"), null, req()).parse());
|
||||
}
|
||||
|
||||
public void testUnknownFunction() throws IOException, SyntaxError {
|
||||
assertSyntaxError("Expecting bad function", "Unknown function in rank query: \"foo\"", () ->
|
||||
getRankQParser(
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "foo"), null, req()).parse());
|
||||
}
|
||||
|
||||
public void testParseLog() throws IOException, SyntaxError {
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(1), 1),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "log",
|
||||
SCALING_FACTOR, "1",
|
||||
WEIGHT, "1"));
|
||||
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(2.5f), 1),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "log",
|
||||
SCALING_FACTOR, "2.5",
|
||||
WEIGHT, "1"));
|
||||
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(1), 2.5f),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "log",
|
||||
SCALING_FACTOR, "1",
|
||||
WEIGHT, "2.5"));
|
||||
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedLogToString(1), 2.5f),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "Log", //use different case
|
||||
SCALING_FACTOR, "1",
|
||||
WEIGHT, "2.5"));
|
||||
}
|
||||
|
||||
public void testParseSigm() throws IOException, SyntaxError {
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSigmoidToString(1.5f, 2f), 1),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "sigm",
|
||||
PIVOT, "1.5",
|
||||
EXPONENT, "2",
|
||||
WEIGHT, "1"));
|
||||
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSigmoidToString(1.5f, 2f), 2),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "sigm",
|
||||
PIVOT, "1.5",
|
||||
EXPONENT, "2",
|
||||
WEIGHT, "2"));
|
||||
}
|
||||
|
||||
public void testParseSatu() throws IOException, SyntaxError {
|
||||
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(1.5f), 1),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "satu",
|
||||
PIVOT, "1.5",
|
||||
WEIGHT, "1"));
|
||||
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(1.5f), 2),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "satu",
|
||||
PIVOT, "1.5",
|
||||
WEIGHT, "2"));
|
||||
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(null), 1),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "satu"));
|
||||
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(null), 1),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "satu",
|
||||
WEIGHT, "1"));
|
||||
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(1.5f), 1),
|
||||
params(FIELD, "rank_1",
|
||||
FUNCTION, "satu",
|
||||
PIVOT, "1.5"));
|
||||
}
|
||||
|
||||
public void testParseDefault() throws IOException, SyntaxError {
|
||||
assertValidRankQuery(expectedFeatureQueryToString("rank_1", expectedSaturationToString(null), 1),
|
||||
params(FIELD, "rank_1"));
|
||||
}
|
||||
|
||||
private void assertValidRankQuery(String expctedToString, SolrParams localParams) throws IOException, SyntaxError {
|
||||
QParser parser = getRankQParser(localParams, null, req());
|
||||
Query q = parser.parse();
|
||||
assertNotNull(q);
|
||||
assertThat(q.toString(), CoreMatchers.equalTo(expctedToString));
|
||||
}
|
||||
|
||||
private String expectedFeatureQueryToString(String fieldName, String function, float boost) {
|
||||
String featureQueryStr = "FeatureQuery(field=" + RankField.INTERNAL_RANK_FIELD_NAME + ", feature=" + fieldName + ", function=" + function + ")";
|
||||
if (boost == 1f) {
|
||||
return featureQueryStr;
|
||||
}
|
||||
return "(" + featureQueryStr + ")^" + boost;
|
||||
}
|
||||
|
||||
private String expectedLogToString(float scalingFactor) {
|
||||
return "LogFunction(scalingFactor=" + scalingFactor + ")";
|
||||
}
|
||||
|
||||
private String expectedSigmoidToString(float pivot, float exp) {
|
||||
return "SigmoidFunction(pivot=" + pivot + ", a=" + exp + ")";
|
||||
}
|
||||
|
||||
private String expectedSaturationToString(Float pivot) {
|
||||
return "SaturationFunction(pivot=" + pivot + ")";
|
||||
}
|
||||
|
||||
private void assertSyntaxError(String assertionMsg, String expectedExceptionMsg, ThrowingRunnable runnable) {
|
||||
SyntaxError se = expectThrows(SyntaxError.class, assertionMsg, runnable);
|
||||
assertThat(se.getMessage(), CoreMatchers.containsString(expectedExceptionMsg));
|
||||
}
|
||||
|
||||
private RankQParser getRankQParser(SolrParams localParams, SolrParams params, SolrQueryRequest req) throws IOException {
|
||||
try (RankQParserPlugin rankQPPlugin = new RankQParserPlugin()) {
|
||||
return (RankQParser) rankQPPlugin.createParser("", localParams, params, req);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -250,6 +250,13 @@
|
|||
|
||||
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
|
||||
<fieldType name="binary" class="solr.BinaryField"/>
|
||||
|
||||
<!--
|
||||
RankFields can be used to store scoring factors to improve document ranking. They should be used
|
||||
in combination with RankQParserPlugin.
|
||||
(experimental)
|
||||
-->
|
||||
<fieldType name="rank" class="solr.RankField"/>
|
||||
|
||||
<!-- solr.TextField allows the specification of custom text analyzers
|
||||
specified as a tokenizer and a list of token filters. Different
|
||||
|
|
|
|||
Loading…
Reference in New Issue