mirror of https://github.com/apache/lucene.git
SOLR-2950: Improve QEC performance by dropping field cache use and keeping a local smaller map
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1220983 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Grant Ingersoll
parent
daa97d0be4
commit
f42b2ffd63
|
|
@ -1,4 +1,6 @@
|
|||
/**
|
||||
package org.apache.lucene.util;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
|
|
@ -15,8 +17,6 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.lucene.search.grouping;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
package org.apache.lucene.util;
|
||||
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
**/
|
||||
public class TestSentinelIntSet extends LuceneTestCase {
|
||||
|
||||
@Test
|
||||
public void test() throws Exception {
|
||||
SentinelIntSet set = new SentinelIntSet(10, -1);
|
||||
assertFalse(set.exists(50));
|
||||
set.put(50);
|
||||
assertTrue(set.exists(50));
|
||||
assertEquals(1, set.size());
|
||||
assertEquals(-11, set.find(10));
|
||||
assertEquals(1, set.size());
|
||||
set.clear();
|
||||
assertEquals(0, set.size());
|
||||
assertEquals(50, set.hash(50));
|
||||
//force a rehash
|
||||
for (int i = 0; i < 20; i++){
|
||||
set.put(i);
|
||||
}
|
||||
assertEquals(20, set.size());
|
||||
assertEquals(24, set.rehashCount);
|
||||
}
|
||||
}
|
||||
|
|
@ -21,7 +21,7 @@ import org.apache.lucene.index.DocValues;
|
|||
import org.apache.lucene.index.DocValues.Type; // javadocs
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
|
||||
import org.apache.lucene.search.grouping.SentinelIntSet;
|
||||
import org.apache.lucene.util.SentinelIntSet;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
import java.io.IOException;
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import org.apache.lucene.index.IndexReader;
|
|||
import org.apache.lucene.search.Sort;
|
||||
import org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector;
|
||||
import org.apache.lucene.search.grouping.SearchGroup;
|
||||
import org.apache.lucene.search.grouping.SentinelIntSet;
|
||||
import org.apache.lucene.util.SentinelIntSet;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
import java.io.IOException;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ package org.apache.lucene.search.grouping.term;
|
|||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
|
||||
import org.apache.lucene.search.grouping.SentinelIntSet;
|
||||
import org.apache.lucene.util.SentinelIntSet;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
import java.io.IOException;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ package org.apache.lucene.search.grouping.term;
|
|||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.search.FieldCache;
|
||||
import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
|
||||
import org.apache.lucene.search.grouping.SentinelIntSet;
|
||||
import org.apache.lucene.util.SentinelIntSet;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
import java.io.IOException;
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import org.apache.lucene.search.FieldCache;
|
|||
import org.apache.lucene.search.Sort;
|
||||
import org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector;
|
||||
import org.apache.lucene.search.grouping.SearchGroup;
|
||||
import org.apache.lucene.search.grouping.SentinelIntSet;
|
||||
import org.apache.lucene.util.SentinelIntSet;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
import java.io.IOException;
|
||||
|
|
|
|||
|
|
@ -234,6 +234,9 @@ Optimizations
|
|||
DirectUpdateHandler2.numDocsPending stats attribute.
|
||||
(Alexey Serba, Mark Miller)
|
||||
|
||||
* SOLR-2950: The QueryElevationComponent now avoids using the FieldCache and looking up
|
||||
every document id (gsingers, yonik)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,46 @@
|
|||
|
||||
package org.apache.solr.handler.component;
|
||||
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.DocumentStoredFieldVisitor;
|
||||
import org.apache.lucene.index.*;
|
||||
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.SentinelIntSet;
|
||||
import org.apache.lucene.util.automaton.Automaton;
|
||||
import org.apache.lucene.util.automaton.CompiledAutomaton;
|
||||
import org.apache.solr.cloud.ZkController;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.QueryElevationParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.DOMUtil;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.core.Config;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.transform.EditorialMarkerFactory;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.search.SortSpec;
|
||||
import org.apache.solr.util.RefCounted;
|
||||
import org.apache.solr.util.VersionedFile;
|
||||
import org.apache.solr.util.plugin.SolrCoreAware;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
|
||||
import javax.xml.xpath.XPath;
|
||||
import javax.xml.xpath.XPathConstants;
|
||||
import javax.xml.xpath.XPathExpressionException;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
|
@ -25,53 +65,12 @@ import java.net.MalformedURLException;
|
|||
import java.net.URL;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.solr.common.params.QueryElevationParams;
|
||||
import org.apache.solr.response.transform.EditorialMarkerFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.xml.xpath.XPath;
|
||||
import javax.xml.xpath.XPathConstants;
|
||||
import javax.xml.xpath.XPathExpressionException;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.solr.cloud.ZkController;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.DOMUtil;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.core.Config;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.schema.StrField;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.search.SortSpec;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.util.VersionedFile;
|
||||
import org.apache.solr.util.RefCounted;
|
||||
import org.apache.solr.util.plugin.SolrCoreAware;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
|
||||
/**
|
||||
* A component to elevate some documents to the top of the result set.
|
||||
*
|
||||
*
|
||||
* @since solr 1.3
|
||||
*/
|
||||
public class QueryElevationComponent extends SearchComponent implements SolrCoreAware
|
||||
{
|
||||
public class QueryElevationComponent extends SearchComponent implements SolrCoreAware {
|
||||
private static Logger log = LoggerFactory.getLogger(QueryElevationComponent.class);
|
||||
|
||||
// Constants used in solrconfig.xml
|
||||
|
|
@ -91,67 +90,58 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
|
|||
// When the configuration is loaded from the data directory.
|
||||
// The key is null if loaded from the config directory, and
|
||||
// is never re-loaded.
|
||||
final Map<IndexReader,Map<String, ElevationObj>> elevationCache =
|
||||
new WeakHashMap<IndexReader, Map<String,ElevationObj>>();
|
||||
final Map<IndexReader, Map<String, ElevationObj>> elevationCache =
|
||||
new WeakHashMap<IndexReader, Map<String, ElevationObj>>();
|
||||
|
||||
class ElevationObj {
|
||||
final String text;
|
||||
final String analyzed;
|
||||
final BooleanClause[] exclude;
|
||||
final BooleanQuery include;
|
||||
final Map<BytesRef,Integer> priority;
|
||||
final Map<BytesRef, Integer> priority;
|
||||
final Set<String> ids;
|
||||
|
||||
// use singletons so hashCode/equals on Sort will just work
|
||||
final FieldComparatorSource comparatorSource;
|
||||
|
||||
ElevationObj( String qstr, List<String> elevate, List<String> exclude ) throws IOException
|
||||
{
|
||||
ElevationObj(String qstr, List<String> elevate, List<String> exclude) throws IOException {
|
||||
this.text = qstr;
|
||||
this.analyzed = getAnalyzedQuery( this.text );
|
||||
this.analyzed = getAnalyzedQuery(this.text);
|
||||
this.ids = new HashSet<String>();
|
||||
|
||||
this.include = new BooleanQuery();
|
||||
this.include.setBoost( 0 );
|
||||
this.include.setBoost(0);
|
||||
this.priority = new HashMap<BytesRef, Integer>();
|
||||
int max = elevate.size()+5;
|
||||
for( String id : elevate ) {
|
||||
int max = elevate.size() + 5;
|
||||
for (String id : elevate) {
|
||||
id = idSchemaFT.readableToIndexed(id);
|
||||
ids.add(id);
|
||||
TermQuery tq = new TermQuery( new Term( idField, id ) );
|
||||
include.add( tq, BooleanClause.Occur.SHOULD );
|
||||
this.priority.put( new BytesRef(id), max-- );
|
||||
TermQuery tq = new TermQuery(new Term(idField, id));
|
||||
include.add(tq, BooleanClause.Occur.SHOULD);
|
||||
this.priority.put(new BytesRef(id), max--);
|
||||
}
|
||||
|
||||
if( exclude == null || exclude.isEmpty() ) {
|
||||
if (exclude == null || exclude.isEmpty()) {
|
||||
this.exclude = null;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
this.exclude = new BooleanClause[exclude.size()];
|
||||
for( int i=0; i<exclude.size(); i++ ) {
|
||||
TermQuery tq = new TermQuery( new Term( idField, idSchemaFT.readableToIndexed(exclude.get(i)) ) );
|
||||
this.exclude[i] = new BooleanClause( tq, BooleanClause.Occur.MUST_NOT );
|
||||
for (int i = 0; i < exclude.size(); i++) {
|
||||
TermQuery tq = new TermQuery(new Term(idField, idSchemaFT.readableToIndexed(exclude.get(i))));
|
||||
this.exclude[i] = new BooleanClause(tq, BooleanClause.Occur.MUST_NOT);
|
||||
}
|
||||
}
|
||||
|
||||
this.comparatorSource = new ElevationComparatorSource(priority);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init( NamedList args )
|
||||
{
|
||||
this.initArgs = SolrParams.toSolrParams( args );
|
||||
public void init(NamedList args) {
|
||||
this.initArgs = SolrParams.toSolrParams(args);
|
||||
}
|
||||
|
||||
public void inform(SolrCore core)
|
||||
{
|
||||
String a = initArgs.get( FIELD_TYPE );
|
||||
if( a != null ) {
|
||||
FieldType ft = core.getSchema().getFieldTypes().get( a );
|
||||
if( ft == null ) {
|
||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Unknown FieldType: '"+a+"' used in QueryElevationComponent" );
|
||||
public void inform(SolrCore core) {
|
||||
String a = initArgs.get(FIELD_TYPE);
|
||||
if (a != null) {
|
||||
FieldType ft = core.getSchema().getFieldTypes().get(a);
|
||||
if (ft == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Unknown FieldType: '" + a + "' used in QueryElevationComponent");
|
||||
}
|
||||
analyzer = ft.getQueryAnalyzer();
|
||||
}
|
||||
|
|
@ -166,154 +156,151 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
|
|||
//register the EditorialMarkerFactory
|
||||
EditorialMarkerFactory factory = new EditorialMarkerFactory();
|
||||
String markerName = initArgs.get(QueryElevationParams.EDITORIAL_MARKER_FIELD_NAME, "elevated");
|
||||
if (markerName == null || markerName.equals("") == true){
|
||||
if (markerName == null || markerName.equals("") == true) {
|
||||
markerName = "elevated";
|
||||
}
|
||||
core.addTransformerFactory(markerName, factory);
|
||||
forceElevation = initArgs.getBool( QueryElevationParams.FORCE_ELEVATION, forceElevation );
|
||||
forceElevation = initArgs.getBool(QueryElevationParams.FORCE_ELEVATION, forceElevation);
|
||||
try {
|
||||
synchronized( elevationCache ) {
|
||||
synchronized (elevationCache) {
|
||||
elevationCache.clear();
|
||||
String f = initArgs.get( CONFIG_FILE );
|
||||
if( f == null ) {
|
||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
|
||||
"QueryElevationComponent must specify argument: '"+CONFIG_FILE
|
||||
+"' -- path to elevate.xml" );
|
||||
String f = initArgs.get(CONFIG_FILE);
|
||||
if (f == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"QueryElevationComponent must specify argument: '" + CONFIG_FILE
|
||||
+ "' -- path to elevate.xml");
|
||||
}
|
||||
boolean exists = false;
|
||||
|
||||
// check if using ZooKeeper
|
||||
ZkController zkController = core.getCoreDescriptor().getCoreContainer().getZkController();
|
||||
if(zkController != null) {
|
||||
if (zkController != null) {
|
||||
// TODO : shouldn't have to keep reading the config name when it has been read before
|
||||
exists = zkController.configFileExists(zkController.readConfigName(core.getCoreDescriptor().getCloudDescriptor().getCollectionName()), f);
|
||||
} else {
|
||||
File fC = new File( core.getResourceLoader().getConfigDir(), f );
|
||||
File fD = new File( core.getDataDir(), f );
|
||||
if( fC.exists() == fD.exists() ) {
|
||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
|
||||
"QueryElevationComponent missing config file: '"+f + "\n"
|
||||
+"either: "+fC.getAbsolutePath() + " or " + fD.getAbsolutePath() + " must exist, but not both." );
|
||||
File fC = new File(core.getResourceLoader().getConfigDir(), f);
|
||||
File fD = new File(core.getDataDir(), f);
|
||||
if (fC.exists() == fD.exists()) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"QueryElevationComponent missing config file: '" + f + "\n"
|
||||
+ "either: " + fC.getAbsolutePath() + " or " + fD.getAbsolutePath() + " must exist, but not both.");
|
||||
}
|
||||
if( fC.exists() ) {
|
||||
if (fC.exists()) {
|
||||
exists = true;
|
||||
log.info( "Loading QueryElevation from: "+ fC.getAbsolutePath() );
|
||||
Config cfg = new Config( core.getResourceLoader(), f );
|
||||
elevationCache.put(null, loadElevationMap( cfg ));
|
||||
log.info("Loading QueryElevation from: " + fC.getAbsolutePath());
|
||||
Config cfg = new Config(core.getResourceLoader(), f);
|
||||
elevationCache.put(null, loadElevationMap(cfg));
|
||||
}
|
||||
}
|
||||
//in other words, we think this is in the data dir, not the conf dir
|
||||
if (!exists){
|
||||
if (!exists) {
|
||||
// preload the first data
|
||||
RefCounted<SolrIndexSearcher> searchHolder = null;
|
||||
try {
|
||||
searchHolder = core.getNewestSearcher(false);
|
||||
IndexReader reader = searchHolder.get().getIndexReader();
|
||||
getElevationMap( reader, core );
|
||||
getElevationMap(reader, core);
|
||||
} finally {
|
||||
if (searchHolder != null) searchHolder.decref();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch( Exception ex ) {
|
||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Error initializing QueryElevationComponent.", ex, false );
|
||||
} catch (Exception ex) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Error initializing QueryElevationComponent.", ex, false);
|
||||
}
|
||||
}
|
||||
|
||||
//get the elevation map from the data dir
|
||||
Map<String, ElevationObj> getElevationMap( IndexReader reader, SolrCore core ) throws Exception
|
||||
{
|
||||
synchronized( elevationCache ) {
|
||||
Map<String, ElevationObj> map = elevationCache.get( null );
|
||||
Map<String, ElevationObj> getElevationMap(IndexReader reader, SolrCore core) throws Exception {
|
||||
synchronized (elevationCache) {
|
||||
Map<String, ElevationObj> map = elevationCache.get(null);
|
||||
if (map != null) return map;
|
||||
|
||||
map = elevationCache.get( reader );
|
||||
if( map == null ) {
|
||||
String f = initArgs.get( CONFIG_FILE );
|
||||
if( f == null ) {
|
||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
|
||||
"QueryElevationComponent must specify argument: "+CONFIG_FILE );
|
||||
map = elevationCache.get(reader);
|
||||
if (map == null) {
|
||||
String f = initArgs.get(CONFIG_FILE);
|
||||
if (f == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"QueryElevationComponent must specify argument: " + CONFIG_FILE);
|
||||
}
|
||||
log.info( "Loading QueryElevation from data dir: "+f );
|
||||
log.info("Loading QueryElevation from data dir: " + f);
|
||||
|
||||
InputStream is = VersionedFile.getLatestFile( core.getDataDir(), f );
|
||||
Config cfg = new Config( core.getResourceLoader(), f, new InputSource(is), null );
|
||||
map = loadElevationMap( cfg );
|
||||
elevationCache.put( reader, map );
|
||||
InputStream is = VersionedFile.getLatestFile(core.getDataDir(), f);
|
||||
Config cfg = new Config(core.getResourceLoader(), f, new InputSource(is), null);
|
||||
map = loadElevationMap(cfg);
|
||||
elevationCache.put(reader, map);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
|
||||
//load up the elevation map
|
||||
private Map<String, ElevationObj> loadElevationMap( Config cfg ) throws IOException
|
||||
{
|
||||
private Map<String, ElevationObj> loadElevationMap(Config cfg) throws IOException {
|
||||
XPath xpath = XPathFactory.newInstance().newXPath();
|
||||
Map<String, ElevationObj> map = new HashMap<String, ElevationObj>();
|
||||
NodeList nodes = (NodeList)cfg.evaluate( "elevate/query", XPathConstants.NODESET );
|
||||
for (int i=0; i<nodes.getLength(); i++) {
|
||||
Node node = nodes.item( i );
|
||||
String qstr = DOMUtil.getAttr( node, "text", "missing query 'text'" );
|
||||
NodeList nodes = (NodeList) cfg.evaluate("elevate/query", XPathConstants.NODESET);
|
||||
for (int i = 0; i < nodes.getLength(); i++) {
|
||||
Node node = nodes.item(i);
|
||||
String qstr = DOMUtil.getAttr(node, "text", "missing query 'text'");
|
||||
|
||||
NodeList children = null;
|
||||
try {
|
||||
children = (NodeList)xpath.evaluate("doc", node, XPathConstants.NODESET);
|
||||
}
|
||||
catch (XPathExpressionException e) {
|
||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
|
||||
"query requires '<doc .../>' child" );
|
||||
children = (NodeList) xpath.evaluate("doc", node, XPathConstants.NODESET);
|
||||
} catch (XPathExpressionException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"query requires '<doc .../>' child");
|
||||
}
|
||||
|
||||
ArrayList<String> include = new ArrayList<String>();
|
||||
ArrayList<String> exclude = new ArrayList<String>();
|
||||
for (int j=0; j<children.getLength(); j++) {
|
||||
for (int j = 0; j < children.getLength(); j++) {
|
||||
Node child = children.item(j);
|
||||
String id = DOMUtil.getAttr( child, "id", "missing 'id'" );
|
||||
String e = DOMUtil.getAttr( child, EXCLUDE, null );
|
||||
if( e != null ) {
|
||||
if( Boolean.valueOf( e ) ) {
|
||||
exclude.add( id );
|
||||
String id = DOMUtil.getAttr(child, "id", "missing 'id'");
|
||||
String e = DOMUtil.getAttr(child, EXCLUDE, null);
|
||||
if (e != null) {
|
||||
if (Boolean.valueOf(e)) {
|
||||
exclude.add(id);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
include.add( id );
|
||||
include.add(id);
|
||||
}
|
||||
|
||||
ElevationObj elev = new ElevationObj( qstr, include, exclude );
|
||||
if( map.containsKey( elev.analyzed ) ) {
|
||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Boosting query defined twice for query: '"+elev.text+"' ("+elev.analyzed+"')" );
|
||||
ElevationObj elev = new ElevationObj(qstr, include, exclude);
|
||||
if (map.containsKey(elev.analyzed)) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Boosting query defined twice for query: '" + elev.text + "' (" + elev.analyzed + "')");
|
||||
}
|
||||
map.put( elev.analyzed, elev );
|
||||
map.put(elev.analyzed, elev);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helpful for testing without loading config.xml
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
void setTopQueryResults( IndexReader reader, String query, String[] ids, String[] ex ) throws IOException
|
||||
{
|
||||
if( ids == null ) {
|
||||
void setTopQueryResults(IndexReader reader, String query, String[] ids, String[] ex) throws IOException {
|
||||
if (ids == null) {
|
||||
ids = new String[0];
|
||||
}
|
||||
if( ex == null ) {
|
||||
if (ex == null) {
|
||||
ex = new String[0];
|
||||
}
|
||||
|
||||
Map<String,ElevationObj> elev = elevationCache.get( reader );
|
||||
if( elev == null ) {
|
||||
Map<String, ElevationObj> elev = elevationCache.get(reader);
|
||||
if (elev == null) {
|
||||
elev = new HashMap<String, ElevationObj>();
|
||||
elevationCache.put( reader, elev );
|
||||
elevationCache.put(reader, elev);
|
||||
}
|
||||
ElevationObj obj = new ElevationObj( query, Arrays.asList(ids), Arrays.asList(ex) );
|
||||
elev.put( obj.analyzed, obj );
|
||||
ElevationObj obj = new ElevationObj(query, Arrays.asList(ids), Arrays.asList(ex));
|
||||
elev.put(obj.analyzed, obj);
|
||||
}
|
||||
|
||||
String getAnalyzedQuery( String query ) throws IOException
|
||||
{
|
||||
if( analyzer == null ) {
|
||||
String getAnalyzedQuery(String query) throws IOException {
|
||||
if (analyzer == null) {
|
||||
return query;
|
||||
}
|
||||
StringBuilder norm = new StringBuilder();
|
||||
|
|
@ -321,8 +308,8 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
|
|||
tokens.reset();
|
||||
|
||||
CharTermAttribute termAtt = tokens.addAttribute(CharTermAttribute.class);
|
||||
while( tokens.incrementToken() ) {
|
||||
norm.append( termAtt.buffer(), 0, termAtt.length() );
|
||||
while (tokens.incrementToken()) {
|
||||
norm.append(termAtt.buffer(), 0, termAtt.length());
|
||||
}
|
||||
tokens.end();
|
||||
tokens.close();
|
||||
|
|
@ -334,22 +321,21 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
|
|||
//---------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
public void prepare(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
public void prepare(ResponseBuilder rb) throws IOException {
|
||||
SolrQueryRequest req = rb.req;
|
||||
SolrParams params = req.getParams();
|
||||
// A runtime param can skip
|
||||
if( !params.getBool( QueryElevationParams.ENABLE, true ) ) {
|
||||
if (!params.getBool(QueryElevationParams.ENABLE, true)) {
|
||||
return;
|
||||
}
|
||||
|
||||
boolean exclusive = params.getBool(QueryElevationParams.EXCLUSIVE, false);
|
||||
// A runtime parameter can alter the config value for forceElevation
|
||||
boolean force = params.getBool( QueryElevationParams.FORCE_ELEVATION, forceElevation );
|
||||
boolean force = params.getBool(QueryElevationParams.FORCE_ELEVATION, forceElevation);
|
||||
|
||||
Query query = rb.getQuery();
|
||||
String qstr = rb.getQueryString();
|
||||
if( query == null || qstr == null) {
|
||||
if (query == null || qstr == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -357,81 +343,80 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
|
|||
IndexReader reader = req.getSearcher().getIndexReader();
|
||||
ElevationObj booster = null;
|
||||
try {
|
||||
booster = getElevationMap( reader, req.getCore() ).get( qstr );
|
||||
}
|
||||
catch( Exception ex ) {
|
||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Error loading elevation", ex );
|
||||
booster = getElevationMap(reader, req.getCore()).get(qstr);
|
||||
} catch (Exception ex) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Error loading elevation", ex);
|
||||
}
|
||||
|
||||
if( booster != null ) {
|
||||
if (booster != null) {
|
||||
rb.req.getContext().put("BOOSTED", booster.ids);
|
||||
|
||||
// Change the query to insert forced documents
|
||||
if (exclusive == true){
|
||||
if (exclusive == true) {
|
||||
//we only want these results
|
||||
rb.setQuery(booster.include);
|
||||
} else {
|
||||
BooleanQuery newq = new BooleanQuery( true );
|
||||
newq.add( query, BooleanClause.Occur.SHOULD );
|
||||
newq.add( booster.include, BooleanClause.Occur.SHOULD );
|
||||
if( booster.exclude != null ) {
|
||||
for( BooleanClause bq : booster.exclude ) {
|
||||
newq.add( bq );
|
||||
BooleanQuery newq = new BooleanQuery(true);
|
||||
newq.add(query, BooleanClause.Occur.SHOULD);
|
||||
newq.add(booster.include, BooleanClause.Occur.SHOULD);
|
||||
if (booster.exclude != null) {
|
||||
for (BooleanClause bq : booster.exclude) {
|
||||
newq.add(bq);
|
||||
}
|
||||
}
|
||||
rb.setQuery( newq );
|
||||
rb.setQuery(newq);
|
||||
}
|
||||
|
||||
|
||||
ElevationComparatorSource comparator = new ElevationComparatorSource(booster);
|
||||
// if the sort is 'score desc' use a custom sorting method to
|
||||
// insert documents in their proper place
|
||||
SortSpec sortSpec = rb.getSortSpec();
|
||||
if( sortSpec.getSort() == null ) {
|
||||
sortSpec.setSort( new Sort(
|
||||
new SortField(idField, booster.comparatorSource, false ),
|
||||
new SortField(null, SortField.Type.SCORE, false)));
|
||||
}
|
||||
else {
|
||||
if (sortSpec.getSort() == null) {
|
||||
sortSpec.setSort(new Sort(new SortField[]{
|
||||
new SortField(idField, comparator, false),
|
||||
new SortField(null, SortField.Type.SCORE, false)
|
||||
}));
|
||||
} else {
|
||||
// Check if the sort is based on score
|
||||
boolean modify = false;
|
||||
SortField[] current = sortSpec.getSort().getSort();
|
||||
ArrayList<SortField> sorts = new ArrayList<SortField>( current.length + 1 );
|
||||
ArrayList<SortField> sorts = new ArrayList<SortField>(current.length + 1);
|
||||
// Perhaps force it to always sort by score
|
||||
if( force && current[0].getType() != SortField.Type.SCORE ) {
|
||||
sorts.add( new SortField(idField, booster.comparatorSource, false ) );
|
||||
if (force && current[0].getType() != SortField.Type.SCORE) {
|
||||
sorts.add(new SortField(idField, comparator, false));
|
||||
modify = true;
|
||||
}
|
||||
for( SortField sf : current ) {
|
||||
if( sf.getType() == SortField.Type.SCORE ) {
|
||||
sorts.add( new SortField(idField, booster.comparatorSource, sf.getReverse() ) );
|
||||
for (SortField sf : current) {
|
||||
if (sf.getType() == SortField.Type.SCORE) {
|
||||
sorts.add(new SortField(idField, comparator, sf.getReverse()));
|
||||
modify = true;
|
||||
}
|
||||
sorts.add( sf );
|
||||
sorts.add(sf);
|
||||
}
|
||||
if( modify ) {
|
||||
sortSpec.setSort( new Sort( sorts.toArray( new SortField[sorts.size()] ) ) );
|
||||
if (modify) {
|
||||
sortSpec.setSort(new Sort(sorts.toArray(new SortField[sorts.size()])));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add debugging information
|
||||
if( rb.isDebug() ) {
|
||||
if (rb.isDebug()) {
|
||||
List<String> match = null;
|
||||
if( booster != null ) {
|
||||
if (booster != null) {
|
||||
// Extract the elevated terms into a list
|
||||
match = new ArrayList<String>(booster.priority.size());
|
||||
for( Object o : booster.include.clauses() ) {
|
||||
TermQuery tq = (TermQuery)((BooleanClause)o).getQuery();
|
||||
match.add( tq.getTerm().text() );
|
||||
for (Object o : booster.include.clauses()) {
|
||||
TermQuery tq = (TermQuery) ((BooleanClause) o).getQuery();
|
||||
match.add(tq.getTerm().text());
|
||||
}
|
||||
}
|
||||
|
||||
SimpleOrderedMap<Object> dbg = new SimpleOrderedMap<Object>();
|
||||
dbg.add( "q", qstr );
|
||||
dbg.add( "match", match );
|
||||
dbg.add("q", qstr);
|
||||
dbg.add("match", match);
|
||||
if (rb.isDebugQuery()) {
|
||||
rb.addDebugInfo("queryBoosting", dbg );
|
||||
rb.addDebugInfo("queryBoosting", dbg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -468,31 +453,33 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
|
|||
@Override
|
||||
public URL[] getDocs() {
|
||||
try {
|
||||
return new URL[] {
|
||||
return new URL[]{
|
||||
new URL("http://wiki.apache.org/solr/QueryElevationComponent")
|
||||
};
|
||||
}
|
||||
catch (MalformedURLException e) {
|
||||
throw new RuntimeException( e );
|
||||
} catch (MalformedURLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
class ElevationComparatorSource extends FieldComparatorSource {
|
||||
private QueryElevationComponent.ElevationObj elevations;
|
||||
private SentinelIntSet ordSet; //the key half of the map
|
||||
private BytesRef[] termValues;//the value half of the map
|
||||
|
||||
class ElevationComparatorSource extends FieldComparatorSource {
|
||||
private final Map<BytesRef,Integer> priority;
|
||||
|
||||
public ElevationComparatorSource( final Map<BytesRef,Integer> boosts) {
|
||||
this.priority = boosts;
|
||||
public ElevationComparatorSource(final QueryElevationComponent.ElevationObj elevations) throws IOException {
|
||||
this.elevations = elevations;
|
||||
int size = elevations.ids.size();
|
||||
ordSet = new SentinelIntSet(size, -1);
|
||||
termValues = new BytesRef[ordSet.keys.length];
|
||||
}
|
||||
|
||||
@Override
|
||||
public FieldComparator<Integer> newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) throws IOException {
|
||||
return new FieldComparator<Integer>() {
|
||||
|
||||
FieldCache.DocTermsIndex idIndex;
|
||||
private final int[] values = new int[numHits];
|
||||
int bottomVal;
|
||||
private final BytesRef tempBR = new BytesRef();
|
||||
private int bottomVal;
|
||||
private TermsEnum termsEnum;
|
||||
private DocsEnum docsEnum;
|
||||
Set<String> seen = new HashSet<String>(elevations.ids.size());
|
||||
|
||||
@Override
|
||||
public int compare(int slot1, int slot2) {
|
||||
|
|
@ -505,10 +492,16 @@ class ElevationComparatorSource extends FieldComparatorSource {
|
|||
}
|
||||
|
||||
private int docVal(int doc) throws IOException {
|
||||
BytesRef id = idIndex.getTerm(doc, tempBR);
|
||||
Integer prio = priority.get(id);
|
||||
if (ordSet.size() > 0) {
|
||||
int slot = ordSet.find(doc);
|
||||
if (slot >= 0) {
|
||||
BytesRef id = termValues[slot];
|
||||
Integer prio = elevations.priority.get(id);
|
||||
return prio == null ? 0 : prio.intValue();
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottom(int doc) throws IOException {
|
||||
|
|
@ -522,7 +515,24 @@ class ElevationComparatorSource extends FieldComparatorSource {
|
|||
|
||||
@Override
|
||||
public FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
|
||||
idIndex = FieldCache.DEFAULT.getTermsIndex(context.reader, fieldname);
|
||||
//convert the ids to Lucene doc ids, the ordSet and termValues needs to be the same size as the number of elevation docs we have
|
||||
ordSet.clear();
|
||||
Fields fields = context.reader.fields();
|
||||
Terms terms = fields.terms(fieldname);
|
||||
termsEnum = terms.iterator(termsEnum);
|
||||
BytesRef term = new BytesRef();
|
||||
|
||||
for (String id : elevations.ids) {
|
||||
term.copyChars(id);
|
||||
if (seen.contains(id) == false && termsEnum.seekExact(term, false)) {
|
||||
docsEnum = termsEnum.docs(null, docsEnum, false);
|
||||
if (docsEnum != null) {
|
||||
int docId = docsEnum.nextDoc();
|
||||
termValues[ordSet.put(docId)] = BytesRef.deepCopyOf(term);
|
||||
seen.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
|
|
@ -533,3 +543,6 @@ class ElevationComparatorSource extends FieldComparatorSource {
|
|||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,13 +17,6 @@
|
|||
|
||||
package org.apache.solr.handler.component;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
|
|
@ -36,11 +29,14 @@ import org.apache.solr.core.SolrCore;
|
|||
import org.apache.solr.handler.component.QueryElevationComponent.ElevationObj;
|
||||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.PrintWriter;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
public class QueryElevationComponentTest extends SolrTestCaseJ4 {
|
||||
|
|
@ -48,7 +44,7 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
|
|||
|
||||
@Before
|
||||
@Override
|
||||
public void setUp() throws Exception{
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
}
|
||||
|
||||
|
|
@ -64,6 +60,7 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
|
|||
File elevateDataFile = new File(dataDir, "elevate-data.xml");
|
||||
FileUtils.copyFile(elevateFile, elevateDataFile);
|
||||
|
||||
|
||||
initCore(config,schema);
|
||||
clearIndex();
|
||||
assertU(commit());
|
||||
|
|
@ -79,29 +76,29 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
|
|||
init("schema11.xml");
|
||||
clearIndex();
|
||||
assertU(commit());
|
||||
assertU(adoc("id", "1", "text", "XXXX XXXX", "str_s", "a" ));
|
||||
assertU(adoc("id", "2", "text", "YYYY", "str_s", "b" ));
|
||||
assertU(adoc("id", "3", "text", "ZZZZ", "str_s", "c" ));
|
||||
assertU(adoc("id", "1", "text", "XXXX XXXX", "str_s", "a"));
|
||||
assertU(adoc("id", "2", "text", "YYYY", "str_s", "b"));
|
||||
assertU(adoc("id", "3", "text", "ZZZZ", "str_s", "c"));
|
||||
|
||||
assertU(adoc("id", "4", "text", "XXXX XXXX", "str_s", "x" ));
|
||||
assertU(adoc("id", "5", "text", "YYYY YYYY", "str_s", "y" ));
|
||||
assertU(adoc("id", "6", "text", "XXXX XXXX", "str_s", "z" ));
|
||||
assertU(adoc("id", "7", "text", "AAAA", "str_s", "a" ));
|
||||
assertU(adoc("id", "8", "text", "AAAA", "str_s", "a" ));
|
||||
assertU(adoc("id", "9", "text", "AAAA AAAA", "str_s", "a" ));
|
||||
assertU(adoc("id", "4", "text", "XXXX XXXX", "str_s", "x"));
|
||||
assertU(adoc("id", "5", "text", "YYYY YYYY", "str_s", "y"));
|
||||
assertU(adoc("id", "6", "text", "XXXX XXXX", "str_s", "z"));
|
||||
assertU(adoc("id", "7", "text", "AAAA", "str_s", "a"));
|
||||
assertU(adoc("id", "8", "text", "AAAA", "str_s", "a"));
|
||||
assertU(adoc("id", "9", "text", "AAAA AAAA", "str_s", "a"));
|
||||
assertU(commit());
|
||||
|
||||
assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate",
|
||||
CommonParams.FL, "id, score, [elevated]")
|
||||
,"//*[@numFound='3']"
|
||||
,"//result/doc[1]/float[@name='id'][.='7.0']"
|
||||
,"//result/doc[2]/float[@name='id'][.='8.0']"
|
||||
,"//result/doc[3]/float[@name='id'][.='9.0']",
|
||||
, "//*[@numFound='3']"
|
||||
, "//result/doc[1]/float[@name='id'][.='7.0']"
|
||||
, "//result/doc[2]/float[@name='id'][.='8.0']"
|
||||
, "//result/doc[3]/float[@name='id'][.='9.0']",
|
||||
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
|
||||
"//result/doc[2]/bool[@name='[elevated]'][.='false']",
|
||||
"//result/doc[3]/bool[@name='[elevated]'][.='false']"
|
||||
);
|
||||
} finally{
|
||||
} finally {
|
||||
delete();
|
||||
}
|
||||
}
|
||||
|
|
@ -141,55 +138,54 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
|
|||
|
||||
|
||||
@Test
|
||||
public void testInterface() throws Exception
|
||||
{
|
||||
public void testInterface() throws Exception {
|
||||
try {
|
||||
init("schema12.xml");
|
||||
SolrCore core = h.getCore();
|
||||
|
||||
NamedList<String> args = new NamedList<String>();
|
||||
args.add( QueryElevationComponent.FIELD_TYPE, "string" );
|
||||
args.add( QueryElevationComponent.CONFIG_FILE, "elevate.xml" );
|
||||
args.add(QueryElevationComponent.FIELD_TYPE, "string");
|
||||
args.add(QueryElevationComponent.CONFIG_FILE, "elevate.xml");
|
||||
|
||||
QueryElevationComponent comp = new QueryElevationComponent();
|
||||
comp.init( args );
|
||||
comp.inform( core );
|
||||
comp.init(args);
|
||||
comp.inform(core);
|
||||
|
||||
SolrQueryRequest req = req();
|
||||
IndexReader reader = req.getSearcher().getIndexReader();
|
||||
Map<String, ElevationObj> map = comp.getElevationMap( reader, core );
|
||||
Map<String, ElevationObj> map = comp.getElevationMap(reader, core);
|
||||
req.close();
|
||||
|
||||
// Make sure the boosts loaded properly
|
||||
assertEquals( 4, map.size() );
|
||||
assertEquals( 1, map.get( "XXXX" ).priority.size() );
|
||||
assertEquals( 2, map.get( "YYYY" ).priority.size() );
|
||||
assertEquals( 3, map.get( "ZZZZ" ).priority.size() );
|
||||
assertEquals( null, map.get( "xxxx" ) );
|
||||
assertEquals( null, map.get( "yyyy" ) );
|
||||
assertEquals( null, map.get( "zzzz" ) );
|
||||
assertEquals(4, map.size());
|
||||
assertEquals(1, map.get("XXXX").priority.size());
|
||||
assertEquals(2, map.get("YYYY").priority.size());
|
||||
assertEquals(3, map.get("ZZZZ").priority.size());
|
||||
assertEquals(null, map.get("xxxx"));
|
||||
assertEquals(null, map.get("yyyy"));
|
||||
assertEquals(null, map.get("zzzz"));
|
||||
|
||||
// Now test the same thing with a lowercase filter: 'lowerfilt'
|
||||
args = new NamedList<String>();
|
||||
args.add( QueryElevationComponent.FIELD_TYPE, "lowerfilt" );
|
||||
args.add( QueryElevationComponent.CONFIG_FILE, "elevate.xml" );
|
||||
args.add(QueryElevationComponent.FIELD_TYPE, "lowerfilt");
|
||||
args.add(QueryElevationComponent.CONFIG_FILE, "elevate.xml");
|
||||
|
||||
comp = new QueryElevationComponent();
|
||||
comp.init( args );
|
||||
comp.inform( core );
|
||||
map = comp.getElevationMap( reader, core );
|
||||
assertEquals( 4, map.size() );
|
||||
assertEquals( null, map.get( "XXXX" ) );
|
||||
assertEquals( null, map.get( "YYYY" ) );
|
||||
assertEquals( null, map.get( "ZZZZ" ) );
|
||||
assertEquals( 1, map.get( "xxxx" ).priority.size() );
|
||||
assertEquals( 2, map.get( "yyyy" ).priority.size() );
|
||||
assertEquals( 3, map.get( "zzzz" ).priority.size() );
|
||||
comp.init(args);
|
||||
comp.inform(core);
|
||||
map = comp.getElevationMap(reader, core);
|
||||
assertEquals(4, map.size());
|
||||
assertEquals(null, map.get("XXXX"));
|
||||
assertEquals(null, map.get("YYYY"));
|
||||
assertEquals(null, map.get("ZZZZ"));
|
||||
assertEquals(1, map.get("xxxx").priority.size());
|
||||
assertEquals(2, map.get("yyyy").priority.size());
|
||||
assertEquals(3, map.get("zzzz").priority.size());
|
||||
|
||||
assertEquals( "xxxx", comp.getAnalyzedQuery( "XXXX" ) );
|
||||
assertEquals( "xxxxyyyy", comp.getAnalyzedQuery( "XXXX YYYY" ) );
|
||||
assertEquals("xxxx", comp.getAnalyzedQuery("XXXX"));
|
||||
assertEquals("xxxxyyyy", comp.getAnalyzedQuery("XXXX YYYY"));
|
||||
|
||||
assertQ("Make sure QEC handles null queries", req("qt","/elevate", "q.alt","*:*", "defType","dismax"),
|
||||
assertQ("Make sure QEC handles null queries", req("qt", "/elevate", "q.alt", "*:*", "defType", "dismax"),
|
||||
"//*[@numFound='0']");
|
||||
} finally {
|
||||
delete();
|
||||
|
|
@ -201,22 +197,22 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
|
|||
public void testMarker() throws Exception {
|
||||
try {
|
||||
init("schema12.xml");
|
||||
assertU(adoc("id", "1", "title", "XXXX XXXX", "str_s1", "a" ));
|
||||
assertU(adoc("id", "2", "title", "YYYY", "str_s1", "b" ));
|
||||
assertU(adoc("id", "3", "title", "ZZZZ", "str_s1", "c" ));
|
||||
assertU(adoc("id", "1", "title", "XXXX XXXX", "str_s1", "a"));
|
||||
assertU(adoc("id", "2", "title", "YYYY", "str_s1", "b"));
|
||||
assertU(adoc("id", "3", "title", "ZZZZ", "str_s1", "c"));
|
||||
|
||||
assertU(adoc("id", "4", "title", "XXXX XXXX", "str_s1", "x" ));
|
||||
assertU(adoc("id", "5", "title", "YYYY YYYY", "str_s1", "y" ));
|
||||
assertU(adoc("id", "6", "title", "XXXX XXXX", "str_s1", "z" ));
|
||||
assertU(adoc("id", "7", "title", "AAAA", "str_s1", "a" ));
|
||||
assertU(adoc("id", "4", "title", "XXXX XXXX", "str_s1", "x"));
|
||||
assertU(adoc("id", "5", "title", "YYYY YYYY", "str_s1", "y"));
|
||||
assertU(adoc("id", "6", "title", "XXXX XXXX", "str_s1", "z"));
|
||||
assertU(adoc("id", "7", "title", "AAAA", "str_s1", "a"));
|
||||
assertU(commit());
|
||||
|
||||
assertQ("", req(CommonParams.Q, "XXXX", CommonParams.QT, "/elevate",
|
||||
CommonParams.FL, "id, score, [elevated]")
|
||||
,"//*[@numFound='3']"
|
||||
,"//result/doc[1]/str[@name='id'][.='1']"
|
||||
,"//result/doc[2]/str[@name='id'][.='4']"
|
||||
,"//result/doc[3]/str[@name='id'][.='6']",
|
||||
, "//*[@numFound='3']"
|
||||
, "//result/doc[1]/str[@name='id'][.='1']"
|
||||
, "//result/doc[2]/str[@name='id'][.='4']"
|
||||
, "//result/doc[3]/str[@name='id'][.='6']",
|
||||
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
|
||||
"//result/doc[2]/bool[@name='[elevated]'][.='false']",
|
||||
"//result/doc[3]/bool[@name='[elevated]'][.='false']"
|
||||
|
|
@ -231,8 +227,8 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
|
|||
|
||||
assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate",
|
||||
CommonParams.FL, "id, score, [elev]")
|
||||
,"//*[@numFound='1']"
|
||||
,"//result/doc[1]/str[@name='id'][.='7']",
|
||||
, "//*[@numFound='1']"
|
||||
, "//result/doc[1]/str[@name='id'][.='7']",
|
||||
"not(//result/doc[1]/bool[@name='[elevated]'][.='false'])",
|
||||
"not(//result/doc[1]/bool[@name='[elev]'][.='false'])" // even though we asked for elev, there is no Transformer registered w/ that, so we shouldn't get a result
|
||||
);
|
||||
|
|
@ -242,115 +238,114 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testSorting() throws Exception
|
||||
{
|
||||
public void testSorting() throws Exception {
|
||||
try {
|
||||
init("schema12.xml");
|
||||
assertU(adoc("id", "a", "title", "ipod", "str_s1", "a" ));
|
||||
assertU(adoc("id", "b", "title", "ipod ipod", "str_s1", "b" ));
|
||||
assertU(adoc("id", "c", "title", "ipod ipod ipod", "str_s1", "c" ));
|
||||
assertU(adoc("id", "a", "title", "ipod", "str_s1", "a"));
|
||||
assertU(adoc("id", "b", "title", "ipod ipod", "str_s1", "b"));
|
||||
assertU(adoc("id", "c", "title", "ipod ipod ipod", "str_s1", "c"));
|
||||
|
||||
assertU(adoc("id", "x", "title", "boosted", "str_s1", "x" ));
|
||||
assertU(adoc("id", "y", "title", "boosted boosted", "str_s1", "y" ));
|
||||
assertU(adoc("id", "z", "title", "boosted boosted boosted", "str_s1", "z" ));
|
||||
assertU(adoc("id", "x", "title", "boosted", "str_s1", "x"));
|
||||
assertU(adoc("id", "y", "title", "boosted boosted", "str_s1", "y"));
|
||||
assertU(adoc("id", "z", "title", "boosted boosted boosted", "str_s1", "z"));
|
||||
assertU(commit());
|
||||
|
||||
String query = "title:ipod";
|
||||
|
||||
Map<String,String> args = new HashMap<String, String>();
|
||||
args.put( CommonParams.Q, query );
|
||||
args.put( CommonParams.QT, "/elevate" );
|
||||
args.put( CommonParams.FL, "id,score" );
|
||||
args.put( "indent", "true" );
|
||||
Map<String, String> args = new HashMap<String, String>();
|
||||
args.put(CommonParams.Q, query);
|
||||
args.put(CommonParams.QT, "/elevate");
|
||||
args.put(CommonParams.FL, "id,score");
|
||||
args.put("indent", "true");
|
||||
//args.put( CommonParams.FL, "id,title,score" );
|
||||
SolrQueryRequest req = new LocalSolrQueryRequest( h.getCore(), new MapSolrParams( args) );
|
||||
SolrQueryRequest req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
|
||||
IndexReader reader = req.getSearcher().getIndexReader();
|
||||
QueryElevationComponent booster = (QueryElevationComponent)req.getCore().getSearchComponent( "elevate" );
|
||||
QueryElevationComponent booster = (QueryElevationComponent) req.getCore().getSearchComponent("elevate");
|
||||
|
||||
assertQ("Make sure standard sort works as expected", req
|
||||
,"//*[@numFound='3']"
|
||||
,"//result/doc[1]/str[@name='id'][.='a']"
|
||||
,"//result/doc[2]/str[@name='id'][.='b']"
|
||||
,"//result/doc[3]/str[@name='id'][.='c']"
|
||||
, "//*[@numFound='3']"
|
||||
, "//result/doc[1]/str[@name='id'][.='a']"
|
||||
, "//result/doc[2]/str[@name='id'][.='b']"
|
||||
, "//result/doc[3]/str[@name='id'][.='c']"
|
||||
);
|
||||
|
||||
// Explicitly set what gets boosted
|
||||
booster.elevationCache.clear();
|
||||
booster.setTopQueryResults( reader, query, new String[] { "x", "y", "z" }, null );
|
||||
booster.setTopQueryResults(reader, query, new String[]{"x", "y", "z"}, null);
|
||||
|
||||
|
||||
assertQ("All six should make it", req
|
||||
,"//*[@numFound='6']"
|
||||
,"//result/doc[1]/str[@name='id'][.='x']"
|
||||
,"//result/doc[2]/str[@name='id'][.='y']"
|
||||
,"//result/doc[3]/str[@name='id'][.='z']"
|
||||
,"//result/doc[4]/str[@name='id'][.='a']"
|
||||
,"//result/doc[5]/str[@name='id'][.='b']"
|
||||
,"//result/doc[6]/str[@name='id'][.='c']"
|
||||
, "//*[@numFound='6']"
|
||||
, "//result/doc[1]/str[@name='id'][.='x']"
|
||||
, "//result/doc[2]/str[@name='id'][.='y']"
|
||||
, "//result/doc[3]/str[@name='id'][.='z']"
|
||||
, "//result/doc[4]/str[@name='id'][.='a']"
|
||||
, "//result/doc[5]/str[@name='id'][.='b']"
|
||||
, "//result/doc[6]/str[@name='id'][.='c']"
|
||||
);
|
||||
|
||||
booster.elevationCache.clear();
|
||||
|
||||
// now switch the order:
|
||||
booster.setTopQueryResults( reader, query, new String[] { "a", "x" }, null );
|
||||
booster.setTopQueryResults(reader, query, new String[]{"a", "x"}, null);
|
||||
assertQ("All four should make it", req
|
||||
,"//*[@numFound='4']"
|
||||
,"//result/doc[1]/str[@name='id'][.='a']"
|
||||
,"//result/doc[2]/str[@name='id'][.='x']"
|
||||
,"//result/doc[3]/str[@name='id'][.='b']"
|
||||
,"//result/doc[4]/str[@name='id'][.='c']"
|
||||
, "//*[@numFound='4']"
|
||||
, "//result/doc[1]/str[@name='id'][.='a']"
|
||||
, "//result/doc[2]/str[@name='id'][.='x']"
|
||||
, "//result/doc[3]/str[@name='id'][.='b']"
|
||||
, "//result/doc[4]/str[@name='id'][.='c']"
|
||||
);
|
||||
|
||||
// Test reverse sort
|
||||
args.put( CommonParams.SORT, "score asc" );
|
||||
args.put(CommonParams.SORT, "score asc");
|
||||
assertQ("All four should make it", req
|
||||
,"//*[@numFound='4']"
|
||||
,"//result/doc[4]/str[@name='id'][.='a']"
|
||||
,"//result/doc[3]/str[@name='id'][.='x']"
|
||||
,"//result/doc[2]/str[@name='id'][.='b']"
|
||||
,"//result/doc[1]/str[@name='id'][.='c']"
|
||||
, "//*[@numFound='4']"
|
||||
, "//result/doc[4]/str[@name='id'][.='a']"
|
||||
, "//result/doc[3]/str[@name='id'][.='x']"
|
||||
, "//result/doc[2]/str[@name='id'][.='b']"
|
||||
, "//result/doc[1]/str[@name='id'][.='c']"
|
||||
);
|
||||
|
||||
// Try normal sort by 'id'
|
||||
// default 'forceBoost' should be false
|
||||
assertEquals( false, booster.forceElevation );
|
||||
args.put( CommonParams.SORT, "str_s1 asc" );
|
||||
assertQ( null, req
|
||||
,"//*[@numFound='4']"
|
||||
,"//result/doc[1]/str[@name='id'][.='a']"
|
||||
,"//result/doc[2]/str[@name='id'][.='b']"
|
||||
,"//result/doc[3]/str[@name='id'][.='c']"
|
||||
,"//result/doc[4]/str[@name='id'][.='x']"
|
||||
assertEquals(false, booster.forceElevation);
|
||||
args.put(CommonParams.SORT, "str_s1 asc");
|
||||
assertQ(null, req
|
||||
, "//*[@numFound='4']"
|
||||
, "//result/doc[1]/str[@name='id'][.='a']"
|
||||
, "//result/doc[2]/str[@name='id'][.='b']"
|
||||
, "//result/doc[3]/str[@name='id'][.='c']"
|
||||
, "//result/doc[4]/str[@name='id'][.='x']"
|
||||
);
|
||||
|
||||
booster.forceElevation = true;
|
||||
assertQ( null, req
|
||||
,"//*[@numFound='4']"
|
||||
,"//result/doc[1]/str[@name='id'][.='a']"
|
||||
,"//result/doc[2]/str[@name='id'][.='x']"
|
||||
,"//result/doc[3]/str[@name='id'][.='b']"
|
||||
,"//result/doc[4]/str[@name='id'][.='c']"
|
||||
assertQ(null, req
|
||||
, "//*[@numFound='4']"
|
||||
, "//result/doc[1]/str[@name='id'][.='a']"
|
||||
, "//result/doc[2]/str[@name='id'][.='x']"
|
||||
, "//result/doc[3]/str[@name='id'][.='b']"
|
||||
, "//result/doc[4]/str[@name='id'][.='c']"
|
||||
);
|
||||
|
||||
//Test exclusive (not to be confused with exclusion)
|
||||
args.put(QueryElevationParams.EXCLUSIVE, "true");
|
||||
booster.setTopQueryResults( reader, query, new String[] { "x", "a" }, new String[] {} );
|
||||
assertQ( null, req
|
||||
,"//*[@numFound='2']"
|
||||
,"//result/doc[1]/str[@name='id'][.='x']"
|
||||
,"//result/doc[2]/str[@name='id'][.='a']"
|
||||
booster.setTopQueryResults(reader, query, new String[]{"x", "a"}, new String[]{});
|
||||
assertQ(null, req
|
||||
, "//*[@numFound='2']"
|
||||
, "//result/doc[1]/str[@name='id'][.='x']"
|
||||
, "//result/doc[2]/str[@name='id'][.='a']"
|
||||
);
|
||||
|
||||
// Test exclusion
|
||||
booster.elevationCache.clear();
|
||||
args.remove( CommonParams.SORT );
|
||||
args.remove( QueryElevationParams.EXCLUSIVE);
|
||||
booster.setTopQueryResults( reader, query, new String[] { "x" }, new String[] { "a" } );
|
||||
assertQ( null, req
|
||||
,"//*[@numFound='3']"
|
||||
,"//result/doc[1]/str[@name='id'][.='x']"
|
||||
,"//result/doc[2]/str[@name='id'][.='b']"
|
||||
,"//result/doc[3]/str[@name='id'][.='c']"
|
||||
args.remove(CommonParams.SORT);
|
||||
args.remove(QueryElevationParams.EXCLUSIVE);
|
||||
booster.setTopQueryResults(reader, query, new String[]{"x"}, new String[]{"a"});
|
||||
assertQ(null, req
|
||||
, "//*[@numFound='3']"
|
||||
, "//result/doc[1]/str[@name='id'][.='x']"
|
||||
, "//result/doc[2]/str[@name='id'][.='b']"
|
||||
, "//result/doc[3]/str[@name='id'][.='c']"
|
||||
);
|
||||
|
||||
|
||||
|
|
@ -361,55 +356,53 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
|
|||
}
|
||||
|
||||
// write a test file to boost some docs
|
||||
private void writeFile( File file, String query, String ... ids ) throws Exception
|
||||
{
|
||||
PrintWriter out = new PrintWriter( new FileOutputStream( file ) );
|
||||
out.println( "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>" );
|
||||
out.println( "<elevate>" );
|
||||
out.println( "<query text=\""+query+"\">" );
|
||||
for( String id : ids ) {
|
||||
out.println( " <doc id=\""+id+"\"/>" );
|
||||
private void writeFile(File file, String query, String... ids) throws Exception {
|
||||
PrintWriter out = new PrintWriter(new FileOutputStream(file));
|
||||
out.println("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>");
|
||||
out.println("<elevate>");
|
||||
out.println("<query text=\"" + query + "\">");
|
||||
for (String id : ids) {
|
||||
out.println(" <doc id=\"" + id + "\"/>");
|
||||
}
|
||||
out.println( "</query>" );
|
||||
out.println( "</elevate>" );
|
||||
out.println("</query>");
|
||||
out.println("</elevate>");
|
||||
out.flush();
|
||||
out.close();
|
||||
|
||||
log.info( "OUT:"+file.getAbsolutePath() );
|
||||
log.info("OUT:" + file.getAbsolutePath());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testElevationReloading() throws Exception
|
||||
{
|
||||
public void testElevationReloading() throws Exception {
|
||||
try {
|
||||
init("schema12.xml");
|
||||
String testfile = "data-elevation.xml";
|
||||
File f = new File( h.getCore().getDataDir(), testfile );
|
||||
writeFile( f, "aaa", "A" );
|
||||
File f = new File(h.getCore().getDataDir(), testfile);
|
||||
writeFile(f, "aaa", "A");
|
||||
|
||||
QueryElevationComponent comp = (QueryElevationComponent)h.getCore().getSearchComponent("elevate");
|
||||
QueryElevationComponent comp = (QueryElevationComponent) h.getCore().getSearchComponent("elevate");
|
||||
NamedList<String> args = new NamedList<String>();
|
||||
args.add( QueryElevationComponent.CONFIG_FILE, testfile );
|
||||
comp.init( args );
|
||||
comp.inform( h.getCore() );
|
||||
args.add(QueryElevationComponent.CONFIG_FILE, testfile);
|
||||
comp.init(args);
|
||||
comp.inform(h.getCore());
|
||||
|
||||
SolrQueryRequest req = req();
|
||||
IndexReader reader = req.getSearcher().getIndexReader();
|
||||
Map<String, ElevationObj> map = comp.getElevationMap(reader, h.getCore());
|
||||
assertTrue( map.get( "aaa" ).priority.containsKey( new BytesRef("A") ) );
|
||||
assertNull( map.get( "bbb" ) );
|
||||
assertTrue(map.get("aaa").priority.containsKey(new BytesRef("A")));
|
||||
assertNull(map.get("bbb"));
|
||||
req.close();
|
||||
|
||||
// now change the file
|
||||
writeFile( f, "bbb", "B" );
|
||||
writeFile(f, "bbb", "B");
|
||||
assertU(adoc("id", "10000")); // will get same reader if no index change
|
||||
assertU(commit());
|
||||
|
||||
req = req();
|
||||
reader = req.getSearcher().getIndexReader();
|
||||
map = comp.getElevationMap(reader, h.getCore());
|
||||
assertNull( map.get( "aaa" ) );
|
||||
assertTrue( map.get( "bbb" ).priority.containsKey( new BytesRef("B") ) );
|
||||
assertNull(map.get("aaa"));
|
||||
assertTrue(map.get("bbb").priority.containsKey(new BytesRef("B")));
|
||||
req.close();
|
||||
} finally {
|
||||
delete();
|
||||
|
|
|
|||
Loading…
Reference in New Issue