Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-2950: Improve QEC performance by dropping field cache use and keeping a local smaller map 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1220983 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Grant Ingersoll 2011-12-19 22:39:01 +00:00
parent daa97d0be4
commit f42b2ffd63
10 changed files with 481 additions and 424 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/grouping/src/java/org/apache/lucene/search/grouping/SentinelIntSet.java → lucene/src/java/org/apache/lucene/util/SentinelIntSet.java
View File

@ -1,4 +1,6 @@
/** package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. * this work for additional information regarding copyright ownership.
@ -15,8 +17,6 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.lucene.search.grouping;
import java.util.Arrays; import java.util.Arrays;
/** /**

48
lucene/src/test/org/apache/lucene/util/TestSentinelIntSet.java Normal file
View File

@ -0,0 +1,48 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.junit.Test;
/**
*
*
**/
public class TestSentinelIntSet extends LuceneTestCase {
@Test
public void test() throws Exception {
SentinelIntSet set = new SentinelIntSet(10, -1);
assertFalse(set.exists(50));
set.put(50);
assertTrue(set.exists(50));
assertEquals(1, set.size());
assertEquals(-11, set.find(10));
assertEquals(1, set.size());
set.clear();
assertEquals(0, set.size());
assertEquals(50, set.hash(50));
//force a rehash
for (int i = 0; i < 20; i++){
set.put(i);
}
assertEquals(20, set.size());
assertEquals(24, set.rehashCount);
}
}

2
modules/grouping/src/java/org/apache/lucene/search/grouping/dv/DVAllGroupsCollector.java
View File

@ -21,7 +21,7 @@ import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValues.Type; // javadocs import org.apache.lucene.index.DocValues.Type; // javadocs
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.grouping.AbstractAllGroupsCollector; import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
import org.apache.lucene.search.grouping.SentinelIntSet; import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import java.io.IOException; import java.io.IOException;

2
modules/grouping/src/java/org/apache/lucene/search/grouping/dv/DVSecondPassGroupingCollector.java
View File

@ -23,7 +23,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Sort; import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector; import org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector;
import org.apache.lucene.search.grouping.SearchGroup; import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.search.grouping.SentinelIntSet; import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import java.io.IOException; import java.io.IOException;

2
modules/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
View File

@ -20,7 +20,7 @@ package org.apache.lucene.search.grouping.term;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.*; import org.apache.lucene.search.*;
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector; import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
import org.apache.lucene.search.grouping.SentinelIntSet; import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import java.io.IOException; import java.io.IOException;

2
modules/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
View File

@ -20,7 +20,7 @@ package org.apache.lucene.search.grouping.term;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.grouping.AbstractAllGroupsCollector; import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
import org.apache.lucene.search.grouping.SentinelIntSet; import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import java.io.IOException; import java.io.IOException;

2
modules/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
View File

@ -22,7 +22,7 @@ import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Sort; import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector; import org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector;
import org.apache.lucene.search.grouping.SearchGroup; import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.search.grouping.SentinelIntSet; import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import java.io.IOException; import java.io.IOException;

3
solr/CHANGES.txt
View File

@ -234,6 +234,9 @@ Optimizations
DirectUpdateHandler2.numDocsPending stats attribute. DirectUpdateHandler2.numDocsPending stats attribute.
(Alexey Serba, Mark Miller) (Alexey Serba, Mark Miller)
* SOLR-2950: The QueryElevationComponent now avoids using the FieldCache and looking up
every document id (gsingers, yonik)
Bug Fixes Bug Fixes
---------------------- ----------------------

491
solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
View File

@ -17,6 +17,46 @@
package org.apache.solr.handler.component; package org.apache.solr.handler.component;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.*;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.*;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.QueryElevationParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.DOMUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.Config;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.transform.EditorialMarkerFactory;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SortSpec;
import org.apache.solr.util.RefCounted;
import org.apache.solr.util.VersionedFile;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
@ -25,60 +65,19 @@ import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.*; import java.util.*;
import org.apache.solr.common.params.QueryElevationParams;
import org.apache.solr.response.transform.EditorialMarkerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.solr.cloud.ZkController;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.DOMUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.Config;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.StrField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SortSpec;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.VersionedFile;
import org.apache.solr.util.RefCounted;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.apache.solr.request.SolrQueryRequest;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
/** /**
* A component to elevate some documents to the top of the result set. * A component to elevate some documents to the top of the result set.
*
* *
* @since solr 1.3 * @since solr 1.3
*/ */
public class QueryElevationComponent extends SearchComponent implements SolrCoreAware public class QueryElevationComponent extends SearchComponent implements SolrCoreAware {
{
private static Logger log = LoggerFactory.getLogger(QueryElevationComponent.class); private static Logger log = LoggerFactory.getLogger(QueryElevationComponent.class);
// Constants used in solrconfig.xml // Constants used in solrconfig.xml
static final String FIELD_TYPE = "queryFieldType"; static final String FIELD_TYPE = "queryFieldType";
static final String CONFIG_FILE = "config-file"; static final String CONFIG_FILE = "config-file";
static final String EXCLUDE = "exclude"; static final String EXCLUDE = "exclude";
// Runtime param -- should be in common? // Runtime param -- should be in common?
private SolrParams initArgs = null; private SolrParams initArgs = null;
@ -91,67 +90,58 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
// When the configuration is loaded from the data directory. // When the configuration is loaded from the data directory.
// The key is null if loaded from the config directory, and // The key is null if loaded from the config directory, and
// is never re-loaded. // is never re-loaded.
final Map<IndexReader,Map<String, ElevationObj>> elevationCache = final Map<IndexReader, Map<String, ElevationObj>> elevationCache =
new WeakHashMap<IndexReader, Map<String,ElevationObj>>(); new WeakHashMap<IndexReader, Map<String, ElevationObj>>();
class ElevationObj { class ElevationObj {
final String text; final String text;
final String analyzed; final String analyzed;
final BooleanClause[] exclude; final BooleanClause[] exclude;
final BooleanQuery include; final BooleanQuery include;
final Map<BytesRef,Integer> priority; final Map<BytesRef, Integer> priority;
final Set<String> ids; final Set<String> ids;
// use singletons so hashCode/equals on Sort will just work
final FieldComparatorSource comparatorSource;
ElevationObj( String qstr, List<String> elevate, List<String> exclude ) throws IOException ElevationObj(String qstr, List<String> elevate, List<String> exclude) throws IOException {
{
this.text = qstr; this.text = qstr;
this.analyzed = getAnalyzedQuery( this.text ); this.analyzed = getAnalyzedQuery(this.text);
this.ids = new HashSet<String>(); this.ids = new HashSet<String>();
this.include = new BooleanQuery(); this.include = new BooleanQuery();
this.include.setBoost( 0 ); this.include.setBoost(0);
this.priority = new HashMap<BytesRef, Integer>(); this.priority = new HashMap<BytesRef, Integer>();
int max = elevate.size()+5; int max = elevate.size() + 5;
for( String id : elevate ) { for (String id : elevate) {
id = idSchemaFT.readableToIndexed(id); id = idSchemaFT.readableToIndexed(id);
ids.add(id); ids.add(id);
TermQuery tq = new TermQuery( new Term( idField, id ) ); TermQuery tq = new TermQuery(new Term(idField, id));
include.add( tq, BooleanClause.Occur.SHOULD ); include.add(tq, BooleanClause.Occur.SHOULD);
this.priority.put( new BytesRef(id), max-- ); this.priority.put(new BytesRef(id), max--);
}
if( exclude == null || exclude.isEmpty() ) {
this.exclude = null;
}
else {
this.exclude = new BooleanClause[exclude.size()];
for( int i=0; i<exclude.size(); i++ ) {
TermQuery tq = new TermQuery( new Term( idField, idSchemaFT.readableToIndexed(exclude.get(i)) ) );
this.exclude[i] = new BooleanClause( tq, BooleanClause.Occur.MUST_NOT );
}
} }
this.comparatorSource = new ElevationComparatorSource(priority); if (exclude == null || exclude.isEmpty()) {
this.exclude = null;
} else {
this.exclude = new BooleanClause[exclude.size()];
for (int i = 0; i < exclude.size(); i++) {
TermQuery tq = new TermQuery(new Term(idField, idSchemaFT.readableToIndexed(exclude.get(i))));
this.exclude[i] = new BooleanClause(tq, BooleanClause.Occur.MUST_NOT);
}
}
} }
} }
@Override @Override
public void init( NamedList args ) public void init(NamedList args) {
{ this.initArgs = SolrParams.toSolrParams(args);
this.initArgs = SolrParams.toSolrParams( args );
} }
public void inform(SolrCore core) public void inform(SolrCore core) {
{ String a = initArgs.get(FIELD_TYPE);
String a = initArgs.get( FIELD_TYPE ); if (a != null) {
if( a != null ) { FieldType ft = core.getSchema().getFieldTypes().get(a);
FieldType ft = core.getSchema().getFieldTypes().get( a ); if (ft == null) {
if( ft == null ) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Unknown FieldType: '" + a + "' used in QueryElevationComponent");
"Unknown FieldType: '"+a+"' used in QueryElevationComponent" );
} }
analyzer = ft.getQueryAnalyzer(); analyzer = ft.getQueryAnalyzer();
} }
@ -166,163 +156,160 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
//register the EditorialMarkerFactory //register the EditorialMarkerFactory
EditorialMarkerFactory factory = new EditorialMarkerFactory(); EditorialMarkerFactory factory = new EditorialMarkerFactory();
String markerName = initArgs.get(QueryElevationParams.EDITORIAL_MARKER_FIELD_NAME, "elevated"); String markerName = initArgs.get(QueryElevationParams.EDITORIAL_MARKER_FIELD_NAME, "elevated");
if (markerName == null || markerName.equals("") == true){ if (markerName == null || markerName.equals("") == true) {
markerName = "elevated"; markerName = "elevated";
} }
core.addTransformerFactory(markerName, factory); core.addTransformerFactory(markerName, factory);
forceElevation = initArgs.getBool( QueryElevationParams.FORCE_ELEVATION, forceElevation ); forceElevation = initArgs.getBool(QueryElevationParams.FORCE_ELEVATION, forceElevation);
try { try {
synchronized( elevationCache ) { synchronized (elevationCache) {
elevationCache.clear(); elevationCache.clear();
String f = initArgs.get( CONFIG_FILE ); String f = initArgs.get(CONFIG_FILE);
if( f == null ) { if (f == null) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"QueryElevationComponent must specify argument: '"+CONFIG_FILE "QueryElevationComponent must specify argument: '" + CONFIG_FILE
+"' -- path to elevate.xml" ); + "' -- path to elevate.xml");
} }
boolean exists = false; boolean exists = false;
// check if using ZooKeeper // check if using ZooKeeper
ZkController zkController = core.getCoreDescriptor().getCoreContainer().getZkController(); ZkController zkController = core.getCoreDescriptor().getCoreContainer().getZkController();
if(zkController != null) { if (zkController != null) {
// TODO : shouldn't have to keep reading the config name when it has been read before // TODO : shouldn't have to keep reading the config name when it has been read before
exists = zkController.configFileExists(zkController.readConfigName(core.getCoreDescriptor().getCloudDescriptor().getCollectionName()), f); exists = zkController.configFileExists(zkController.readConfigName(core.getCoreDescriptor().getCloudDescriptor().getCollectionName()), f);
} else { } else {
File fC = new File( core.getResourceLoader().getConfigDir(), f ); File fC = new File(core.getResourceLoader().getConfigDir(), f);
File fD = new File( core.getDataDir(), f ); File fD = new File(core.getDataDir(), f);
if( fC.exists() == fD.exists() ) { if (fC.exists() == fD.exists()) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"QueryElevationComponent missing config file: '"+f + "\n" "QueryElevationComponent missing config file: '" + f + "\n"
+"either: "+fC.getAbsolutePath() + " or " + fD.getAbsolutePath() + " must exist, but not both." ); + "either: " + fC.getAbsolutePath() + " or " + fD.getAbsolutePath() + " must exist, but not both.");
} }
if( fC.exists() ) { if (fC.exists()) {
exists = true; exists = true;
log.info( "Loading QueryElevation from: "+ fC.getAbsolutePath() ); log.info("Loading QueryElevation from: " + fC.getAbsolutePath());
Config cfg = new Config( core.getResourceLoader(), f ); Config cfg = new Config(core.getResourceLoader(), f);
elevationCache.put(null, loadElevationMap( cfg )); elevationCache.put(null, loadElevationMap(cfg));
} }
} }
//in other words, we think this is in the data dir, not the conf dir //in other words, we think this is in the data dir, not the conf dir
if (!exists){ if (!exists) {
// preload the first data // preload the first data
RefCounted<SolrIndexSearcher> searchHolder = null; RefCounted<SolrIndexSearcher> searchHolder = null;
try { try {
searchHolder = core.getNewestSearcher(false); searchHolder = core.getNewestSearcher(false);
IndexReader reader = searchHolder.get().getIndexReader(); IndexReader reader = searchHolder.get().getIndexReader();
getElevationMap( reader, core ); getElevationMap(reader, core);
} finally { } finally {
if (searchHolder != null) searchHolder.decref(); if (searchHolder != null) searchHolder.decref();
} }
} }
} }
} } catch (Exception ex) {
catch( Exception ex ) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error initializing QueryElevationComponent.", ex, false);
"Error initializing QueryElevationComponent.", ex, false );
} }
} }
//get the elevation map from the data dir //get the elevation map from the data dir
Map<String, ElevationObj> getElevationMap( IndexReader reader, SolrCore core ) throws Exception Map<String, ElevationObj> getElevationMap(IndexReader reader, SolrCore core) throws Exception {
{ synchronized (elevationCache) {
synchronized( elevationCache ) { Map<String, ElevationObj> map = elevationCache.get(null);
Map<String, ElevationObj> map = elevationCache.get( null );
if (map != null) return map; if (map != null) return map;
map = elevationCache.get( reader ); map = elevationCache.get(reader);
if( map == null ) { if (map == null) {
String f = initArgs.get( CONFIG_FILE ); String f = initArgs.get(CONFIG_FILE);
if( f == null ) { if (f == null) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"QueryElevationComponent must specify argument: "+CONFIG_FILE ); "QueryElevationComponent must specify argument: " + CONFIG_FILE);
} }
log.info( "Loading QueryElevation from data dir: "+f ); log.info("Loading QueryElevation from data dir: " + f);
InputStream is = VersionedFile.getLatestFile( core.getDataDir(), f ); InputStream is = VersionedFile.getLatestFile(core.getDataDir(), f);
Config cfg = new Config( core.getResourceLoader(), f, new InputSource(is), null ); Config cfg = new Config(core.getResourceLoader(), f, new InputSource(is), null);
map = loadElevationMap( cfg ); map = loadElevationMap(cfg);
elevationCache.put( reader, map ); elevationCache.put(reader, map);
} }
return map; return map;
} }
} }
//load up the elevation map //load up the elevation map
private Map<String, ElevationObj> loadElevationMap( Config cfg ) throws IOException private Map<String, ElevationObj> loadElevationMap(Config cfg) throws IOException {
{
XPath xpath = XPathFactory.newInstance().newXPath(); XPath xpath = XPathFactory.newInstance().newXPath();
Map<String, ElevationObj> map = new HashMap<String, ElevationObj>(); Map<String, ElevationObj> map = new HashMap<String, ElevationObj>();
NodeList nodes = (NodeList)cfg.evaluate( "elevate/query", XPathConstants.NODESET ); NodeList nodes = (NodeList) cfg.evaluate("elevate/query", XPathConstants.NODESET);
for (int i=0; i<nodes.getLength(); i++) { for (int i = 0; i < nodes.getLength(); i++) {
Node node = nodes.item( i ); Node node = nodes.item(i);
String qstr = DOMUtil.getAttr( node, "text", "missing query 'text'" ); String qstr = DOMUtil.getAttr(node, "text", "missing query 'text'");
NodeList children = null; NodeList children = null;
try { try {
children = (NodeList)xpath.evaluate("doc", node, XPathConstants.NODESET); children = (NodeList) xpath.evaluate("doc", node, XPathConstants.NODESET);
} } catch (XPathExpressionException e) {
catch (XPathExpressionException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "query requires '<doc .../>' child");
"query requires '<doc .../>' child" );
} }
ArrayList<String> include = new ArrayList<String>(); ArrayList<String> include = new ArrayList<String>();
ArrayList<String> exclude = new ArrayList<String>(); ArrayList<String> exclude = new ArrayList<String>();
for (int j=0; j<children.getLength(); j++) { for (int j = 0; j < children.getLength(); j++) {
Node child = children.item(j); Node child = children.item(j);
String id = DOMUtil.getAttr( child, "id", "missing 'id'" ); String id = DOMUtil.getAttr(child, "id", "missing 'id'");
String e = DOMUtil.getAttr( child, EXCLUDE, null ); String e = DOMUtil.getAttr(child, EXCLUDE, null);
if( e != null ) { if (e != null) {
if( Boolean.valueOf( e ) ) { if (Boolean.valueOf(e)) {
exclude.add( id ); exclude.add(id);
continue; continue;
} }
} }
include.add( id ); include.add(id);
} }
ElevationObj elev = new ElevationObj( qstr, include, exclude ); ElevationObj elev = new ElevationObj(qstr, include, exclude);
if( map.containsKey( elev.analyzed ) ) { if (map.containsKey(elev.analyzed)) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Boosting query defined twice for query: '"+elev.text+"' ("+elev.analyzed+"')" ); "Boosting query defined twice for query: '" + elev.text + "' (" + elev.analyzed + "')");
} }
map.put( elev.analyzed, elev ); map.put(elev.analyzed, elev);
} }
return map; return map;
} }
/** /**
* Helpful for testing without loading config.xml * Helpful for testing without loading config.xml
* @throws IOException *
* @throws IOException
*/ */
void setTopQueryResults( IndexReader reader, String query, String[] ids, String[] ex ) throws IOException void setTopQueryResults(IndexReader reader, String query, String[] ids, String[] ex) throws IOException {
{ if (ids == null) {
if( ids == null ) {
ids = new String[0]; ids = new String[0];
} }
if( ex == null ) { if (ex == null) {
ex = new String[0]; ex = new String[0];
} }
Map<String,ElevationObj> elev = elevationCache.get( reader ); Map<String, ElevationObj> elev = elevationCache.get(reader);
if( elev == null ) { if (elev == null) {
elev = new HashMap<String, ElevationObj>(); elev = new HashMap<String, ElevationObj>();
elevationCache.put( reader, elev ); elevationCache.put(reader, elev);
} }
ElevationObj obj = new ElevationObj( query, Arrays.asList(ids), Arrays.asList(ex) ); ElevationObj obj = new ElevationObj(query, Arrays.asList(ids), Arrays.asList(ex));
elev.put( obj.analyzed, obj ); elev.put(obj.analyzed, obj);
} }
String getAnalyzedQuery( String query ) throws IOException String getAnalyzedQuery(String query) throws IOException {
{ if (analyzer == null) {
if( analyzer == null ) {
return query; return query;
} }
StringBuilder norm = new StringBuilder(); StringBuilder norm = new StringBuilder();
TokenStream tokens = analyzer.tokenStream("", new StringReader(query)); TokenStream tokens = analyzer.tokenStream("", new StringReader(query));
tokens.reset(); tokens.reset();
CharTermAttribute termAtt = tokens.addAttribute(CharTermAttribute.class); CharTermAttribute termAtt = tokens.addAttribute(CharTermAttribute.class);
while( tokens.incrementToken() ) { while (tokens.incrementToken()) {
norm.append( termAtt.buffer(), 0, termAtt.length() ); norm.append(termAtt.buffer(), 0, termAtt.length());
} }
tokens.end(); tokens.end();
tokens.close(); tokens.close();
@ -332,24 +319,23 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
//--------------------------------------------------------------------------------- //---------------------------------------------------------------------------------
// SearchComponent // SearchComponent
//--------------------------------------------------------------------------------- //---------------------------------------------------------------------------------
@Override @Override
public void prepare(ResponseBuilder rb) throws IOException public void prepare(ResponseBuilder rb) throws IOException {
{
SolrQueryRequest req = rb.req; SolrQueryRequest req = rb.req;
SolrParams params = req.getParams(); SolrParams params = req.getParams();
// A runtime param can skip // A runtime param can skip
if( !params.getBool( QueryElevationParams.ENABLE, true ) ) { if (!params.getBool(QueryElevationParams.ENABLE, true)) {
return; return;
} }
boolean exclusive = params.getBool(QueryElevationParams.EXCLUSIVE, false); boolean exclusive = params.getBool(QueryElevationParams.EXCLUSIVE, false);
// A runtime parameter can alter the config value for forceElevation // A runtime parameter can alter the config value for forceElevation
boolean force = params.getBool( QueryElevationParams.FORCE_ELEVATION, forceElevation ); boolean force = params.getBool(QueryElevationParams.FORCE_ELEVATION, forceElevation);
Query query = rb.getQuery(); Query query = rb.getQuery();
String qstr = rb.getQueryString(); String qstr = rb.getQueryString();
if( query == null || qstr == null) { if (query == null || qstr == null) {
return; return;
} }
@ -357,81 +343,80 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
IndexReader reader = req.getSearcher().getIndexReader(); IndexReader reader = req.getSearcher().getIndexReader();
ElevationObj booster = null; ElevationObj booster = null;
try { try {
booster = getElevationMap( reader, req.getCore() ).get( qstr ); booster = getElevationMap(reader, req.getCore()).get(qstr);
} catch (Exception ex) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error loading elevation", ex);
} }
catch( Exception ex ) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, if (booster != null) {
"Error loading elevation", ex );
}
if( booster != null ) {
rb.req.getContext().put("BOOSTED", booster.ids); rb.req.getContext().put("BOOSTED", booster.ids);
// Change the query to insert forced documents // Change the query to insert forced documents
if (exclusive == true){ if (exclusive == true) {
//we only want these results //we only want these results
rb.setQuery(booster.include); rb.setQuery(booster.include);
} else { } else {
BooleanQuery newq = new BooleanQuery( true ); BooleanQuery newq = new BooleanQuery(true);
newq.add( query, BooleanClause.Occur.SHOULD ); newq.add(query, BooleanClause.Occur.SHOULD);
newq.add( booster.include, BooleanClause.Occur.SHOULD ); newq.add(booster.include, BooleanClause.Occur.SHOULD);
if( booster.exclude != null ) { if (booster.exclude != null) {
for( BooleanClause bq : booster.exclude ) { for (BooleanClause bq : booster.exclude) {
newq.add( bq ); newq.add(bq);
} }
} }
rb.setQuery( newq ); rb.setQuery(newq);
} }
ElevationComparatorSource comparator = new ElevationComparatorSource(booster);
// if the sort is 'score desc' use a custom sorting method to // if the sort is 'score desc' use a custom sorting method to
// insert documents in their proper place // insert documents in their proper place
SortSpec sortSpec = rb.getSortSpec(); SortSpec sortSpec = rb.getSortSpec();
if( sortSpec.getSort() == null ) { if (sortSpec.getSort() == null) {
sortSpec.setSort( new Sort( sortSpec.setSort(new Sort(new SortField[]{
new SortField(idField, booster.comparatorSource, false ), new SortField(idField, comparator, false),
new SortField(null, SortField.Type.SCORE, false))); new SortField(null, SortField.Type.SCORE, false)
} }));
else { } else {
// Check if the sort is based on score // Check if the sort is based on score
boolean modify = false; boolean modify = false;
SortField[] current = sortSpec.getSort().getSort(); SortField[] current = sortSpec.getSort().getSort();
ArrayList<SortField> sorts = new ArrayList<SortField>( current.length + 1 ); ArrayList<SortField> sorts = new ArrayList<SortField>(current.length + 1);
// Perhaps force it to always sort by score // Perhaps force it to always sort by score
if( force && current[0].getType() != SortField.Type.SCORE ) { if (force && current[0].getType() != SortField.Type.SCORE) {
sorts.add( new SortField(idField, booster.comparatorSource, false ) ); sorts.add(new SortField(idField, comparator, false));
modify = true; modify = true;
} }
for( SortField sf : current ) { for (SortField sf : current) {
if( sf.getType() == SortField.Type.SCORE ) { if (sf.getType() == SortField.Type.SCORE) {
sorts.add( new SortField(idField, booster.comparatorSource, sf.getReverse() ) ); sorts.add(new SortField(idField, comparator, sf.getReverse()));
modify = true; modify = true;
} }
sorts.add( sf ); sorts.add(sf);
} }
if( modify ) { if (modify) {
sortSpec.setSort( new Sort( sorts.toArray( new SortField[sorts.size()] ) ) ); sortSpec.setSort(new Sort(sorts.toArray(new SortField[sorts.size()])));
} }
} }
} }
// Add debugging information // Add debugging information
if( rb.isDebug() ) { if (rb.isDebug()) {
List<String> match = null; List<String> match = null;
if( booster != null ) { if (booster != null) {
// Extract the elevated terms into a list // Extract the elevated terms into a list
match = new ArrayList<String>(booster.priority.size()); match = new ArrayList<String>(booster.priority.size());
for( Object o : booster.include.clauses() ) { for (Object o : booster.include.clauses()) {
TermQuery tq = (TermQuery)((BooleanClause)o).getQuery(); TermQuery tq = (TermQuery) ((BooleanClause) o).getQuery();
match.add( tq.getTerm().text() ); match.add(tq.getTerm().text());
} }
} }
SimpleOrderedMap<Object> dbg = new SimpleOrderedMap<Object>(); SimpleOrderedMap<Object> dbg = new SimpleOrderedMap<Object>();
dbg.add( "q", qstr ); dbg.add("q", qstr);
dbg.add( "match", match ); dbg.add("match", match);
if (rb.isDebugQuery()) { if (rb.isDebugQuery()) {
rb.addDebugInfo("queryBoosting", dbg ); rb.addDebugInfo("queryBoosting", dbg);
} }
} }
} }
@ -440,7 +425,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
public void process(ResponseBuilder rb) throws IOException { public void process(ResponseBuilder rb) throws IOException {
// Do nothing -- the real work is modifying the input query // Do nothing -- the real work is modifying the input query
} }
//--------------------------------------------------------------------------------- //---------------------------------------------------------------------------------
// SolrInfoMBean // SolrInfoMBean
//--------------------------------------------------------------------------------- //---------------------------------------------------------------------------------
@ -468,31 +453,33 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
@Override @Override
public URL[] getDocs() { public URL[] getDocs() {
try { try {
return new URL[] { return new URL[]{
new URL("http://wiki.apache.org/solr/QueryElevationComponent") new URL("http://wiki.apache.org/solr/QueryElevationComponent")
}; };
} } catch (MalformedURLException e) {
catch (MalformedURLException e) { throw new RuntimeException(e);
throw new RuntimeException( e );
} }
} }
} class ElevationComparatorSource extends FieldComparatorSource {
private QueryElevationComponent.ElevationObj elevations;
private SentinelIntSet ordSet; //the key half of the map
private BytesRef[] termValues;//the value half of the map
class ElevationComparatorSource extends FieldComparatorSource { public ElevationComparatorSource(final QueryElevationComponent.ElevationObj elevations) throws IOException {
private final Map<BytesRef,Integer> priority; this.elevations = elevations;
int size = elevations.ids.size();
public ElevationComparatorSource( final Map<BytesRef,Integer> boosts) { ordSet = new SentinelIntSet(size, -1);
this.priority = boosts; termValues = new BytesRef[ordSet.keys.length];
} }
@Override @Override
public FieldComparator<Integer> newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) throws IOException { public FieldComparator<Integer> newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) throws IOException {
return new FieldComparator<Integer>() { return new FieldComparator<Integer>() {
FieldCache.DocTermsIndex idIndex;
private final int[] values = new int[numHits]; private final int[] values = new int[numHits];
int bottomVal; private int bottomVal;
private final BytesRef tempBR = new BytesRef(); private TermsEnum termsEnum;
private DocsEnum docsEnum;
Set<String> seen = new HashSet<String>(elevations.ids.size());
@Override @Override
public int compare(int slot1, int slot2) { public int compare(int slot1, int slot2) {
@ -505,9 +492,15 @@ class ElevationComparatorSource extends FieldComparatorSource {
} }
private int docVal(int doc) throws IOException { private int docVal(int doc) throws IOException {
BytesRef id = idIndex.getTerm(doc, tempBR); if (ordSet.size() > 0) {
Integer prio = priority.get(id); int slot = ordSet.find(doc);
return prio == null ? 0 : prio.intValue(); if (slot >= 0) {
BytesRef id = termValues[slot];
Integer prio = elevations.priority.get(id);
return prio == null ? 0 : prio.intValue();
}
}
return 0;
} }
@Override @Override
@ -522,7 +515,24 @@ class ElevationComparatorSource extends FieldComparatorSource {
@Override @Override
public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { public FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
idIndex = FieldCache.DEFAULT.getTermsIndex(context.reader, fieldname); //convert the ids to Lucene doc ids, the ordSet and termValues needs to be the same size as the number of elevation docs we have
ordSet.clear();
Fields fields = context.reader.fields();
Terms terms = fields.terms(fieldname);
termsEnum = terms.iterator(termsEnum);
BytesRef term = new BytesRef();
for (String id : elevations.ids) {
term.copyChars(id);
if (seen.contains(id) == false && termsEnum.seekExact(term, false)) {
docsEnum = termsEnum.docs(null, docsEnum, false);
if (docsEnum != null) {
int docId = docsEnum.nextDoc();
termValues[ordSet.put(docId)] = BytesRef.deepCopyOf(term);
seen.add(id);
}
}
}
return this; return this;
} }
@ -533,3 +543,6 @@ class ElevationComparatorSource extends FieldComparatorSource {
}; };
} }
} }
}

347
solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
View File

@ -17,13 +17,6 @@
package org.apache.solr.handler.component; package org.apache.solr.handler.component;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4;
@ -36,11 +29,14 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.QueryElevationComponent.ElevationObj; import org.apache.solr.handler.component.QueryElevationComponent.ElevationObj;
import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.io.PrintWriter;
import java.util.HashMap;
import java.util.Map;
public class QueryElevationComponentTest extends SolrTestCaseJ4 { public class QueryElevationComponentTest extends SolrTestCaseJ4 {
@ -48,7 +44,7 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
@Before @Before
@Override @Override
public void setUp() throws Exception{ public void setUp() throws Exception {
super.setUp(); super.setUp();
} }
@ -64,6 +60,7 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
File elevateDataFile = new File(dataDir, "elevate-data.xml"); File elevateDataFile = new File(dataDir, "elevate-data.xml");
FileUtils.copyFile(elevateFile, elevateDataFile); FileUtils.copyFile(elevateFile, elevateDataFile);
initCore(config,schema); initCore(config,schema);
clearIndex(); clearIndex();
assertU(commit()); assertU(commit());
@ -79,29 +76,29 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
init("schema11.xml"); init("schema11.xml");
clearIndex(); clearIndex();
assertU(commit()); assertU(commit());
assertU(adoc("id", "1", "text", "XXXX XXXX", "str_s", "a" )); assertU(adoc("id", "1", "text", "XXXX XXXX", "str_s", "a"));
assertU(adoc("id", "2", "text", "YYYY", "str_s", "b" )); assertU(adoc("id", "2", "text", "YYYY", "str_s", "b"));
assertU(adoc("id", "3", "text", "ZZZZ", "str_s", "c" )); assertU(adoc("id", "3", "text", "ZZZZ", "str_s", "c"));
assertU(adoc("id", "4", "text", "XXXX XXXX", "str_s", "x" )); assertU(adoc("id", "4", "text", "XXXX XXXX", "str_s", "x"));
assertU(adoc("id", "5", "text", "YYYY YYYY", "str_s", "y" )); assertU(adoc("id", "5", "text", "YYYY YYYY", "str_s", "y"));
assertU(adoc("id", "6", "text", "XXXX XXXX", "str_s", "z" )); assertU(adoc("id", "6", "text", "XXXX XXXX", "str_s", "z"));
assertU(adoc("id", "7", "text", "AAAA", "str_s", "a" )); assertU(adoc("id", "7", "text", "AAAA", "str_s", "a"));
assertU(adoc("id", "8", "text", "AAAA", "str_s", "a" )); assertU(adoc("id", "8", "text", "AAAA", "str_s", "a"));
assertU(adoc("id", "9", "text", "AAAA AAAA", "str_s", "a" )); assertU(adoc("id", "9", "text", "AAAA AAAA", "str_s", "a"));
assertU(commit()); assertU(commit());
assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate", assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate",
CommonParams.FL, "id, score, [elevated]") CommonParams.FL, "id, score, [elevated]")
,"//*[@numFound='3']" , "//*[@numFound='3']"
,"//result/doc[1]/float[@name='id'][.='7.0']" , "//result/doc[1]/float[@name='id'][.='7.0']"
,"//result/doc[2]/float[@name='id'][.='8.0']" , "//result/doc[2]/float[@name='id'][.='8.0']"
,"//result/doc[3]/float[@name='id'][.='9.0']", , "//result/doc[3]/float[@name='id'][.='9.0']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']", "//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='false']", "//result/doc[2]/bool[@name='[elevated]'][.='false']",
"//result/doc[3]/bool[@name='[elevated]'][.='false']" "//result/doc[3]/bool[@name='[elevated]'][.='false']"
); );
} finally{ } finally {
delete(); delete();
} }
} }
@ -141,55 +138,54 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
@Test @Test
public void testInterface() throws Exception public void testInterface() throws Exception {
{
try { try {
init("schema12.xml"); init("schema12.xml");
SolrCore core = h.getCore(); SolrCore core = h.getCore();
NamedList<String> args = new NamedList<String>(); NamedList<String> args = new NamedList<String>();
args.add( QueryElevationComponent.FIELD_TYPE, "string" ); args.add(QueryElevationComponent.FIELD_TYPE, "string");
args.add( QueryElevationComponent.CONFIG_FILE, "elevate.xml" ); args.add(QueryElevationComponent.CONFIG_FILE, "elevate.xml");
QueryElevationComponent comp = new QueryElevationComponent(); QueryElevationComponent comp = new QueryElevationComponent();
comp.init( args ); comp.init(args);
comp.inform( core ); comp.inform(core);
SolrQueryRequest req = req(); SolrQueryRequest req = req();
IndexReader reader = req.getSearcher().getIndexReader(); IndexReader reader = req.getSearcher().getIndexReader();
Map<String, ElevationObj> map = comp.getElevationMap( reader, core ); Map<String, ElevationObj> map = comp.getElevationMap(reader, core);
req.close(); req.close();
// Make sure the boosts loaded properly // Make sure the boosts loaded properly
assertEquals( 4, map.size() ); assertEquals(4, map.size());
assertEquals( 1, map.get( "XXXX" ).priority.size() ); assertEquals(1, map.get("XXXX").priority.size());
assertEquals( 2, map.get( "YYYY" ).priority.size() ); assertEquals(2, map.get("YYYY").priority.size());
assertEquals( 3, map.get( "ZZZZ" ).priority.size() ); assertEquals(3, map.get("ZZZZ").priority.size());
assertEquals( null, map.get( "xxxx" ) ); assertEquals(null, map.get("xxxx"));
assertEquals( null, map.get( "yyyy" ) ); assertEquals(null, map.get("yyyy"));
assertEquals( null, map.get( "zzzz" ) ); assertEquals(null, map.get("zzzz"));
// Now test the same thing with a lowercase filter: 'lowerfilt' // Now test the same thing with a lowercase filter: 'lowerfilt'
args = new NamedList<String>(); args = new NamedList<String>();
args.add( QueryElevationComponent.FIELD_TYPE, "lowerfilt" ); args.add(QueryElevationComponent.FIELD_TYPE, "lowerfilt");
args.add( QueryElevationComponent.CONFIG_FILE, "elevate.xml" ); args.add(QueryElevationComponent.CONFIG_FILE, "elevate.xml");
comp = new QueryElevationComponent(); comp = new QueryElevationComponent();
comp.init( args ); comp.init(args);
comp.inform( core ); comp.inform(core);
map = comp.getElevationMap( reader, core ); map = comp.getElevationMap(reader, core);
assertEquals( 4, map.size() ); assertEquals(4, map.size());
assertEquals( null, map.get( "XXXX" ) ); assertEquals(null, map.get("XXXX"));
assertEquals( null, map.get( "YYYY" ) ); assertEquals(null, map.get("YYYY"));
assertEquals( null, map.get( "ZZZZ" ) ); assertEquals(null, map.get("ZZZZ"));
assertEquals( 1, map.get( "xxxx" ).priority.size() ); assertEquals(1, map.get("xxxx").priority.size());
assertEquals( 2, map.get( "yyyy" ).priority.size() ); assertEquals(2, map.get("yyyy").priority.size());
assertEquals( 3, map.get( "zzzz" ).priority.size() ); assertEquals(3, map.get("zzzz").priority.size());
assertEquals( "xxxx", comp.getAnalyzedQuery( "XXXX" ) ); assertEquals("xxxx", comp.getAnalyzedQuery("XXXX"));
assertEquals( "xxxxyyyy", comp.getAnalyzedQuery( "XXXX YYYY" ) ); assertEquals("xxxxyyyy", comp.getAnalyzedQuery("XXXX YYYY"));
assertQ("Make sure QEC handles null queries", req("qt","/elevate", "q.alt","*:*", "defType","dismax"), assertQ("Make sure QEC handles null queries", req("qt", "/elevate", "q.alt", "*:*", "defType", "dismax"),
"//*[@numFound='0']"); "//*[@numFound='0']");
} finally { } finally {
delete(); delete();
@ -201,26 +197,26 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
public void testMarker() throws Exception { public void testMarker() throws Exception {
try { try {
init("schema12.xml"); init("schema12.xml");
assertU(adoc("id", "1", "title", "XXXX XXXX", "str_s1", "a" )); assertU(adoc("id", "1", "title", "XXXX XXXX", "str_s1", "a"));
assertU(adoc("id", "2", "title", "YYYY", "str_s1", "b" )); assertU(adoc("id", "2", "title", "YYYY", "str_s1", "b"));
assertU(adoc("id", "3", "title", "ZZZZ", "str_s1", "c" )); assertU(adoc("id", "3", "title", "ZZZZ", "str_s1", "c"));
assertU(adoc("id", "4", "title", "XXXX XXXX", "str_s1", "x" )); assertU(adoc("id", "4", "title", "XXXX XXXX", "str_s1", "x"));
assertU(adoc("id", "5", "title", "YYYY YYYY", "str_s1", "y" )); assertU(adoc("id", "5", "title", "YYYY YYYY", "str_s1", "y"));
assertU(adoc("id", "6", "title", "XXXX XXXX", "str_s1", "z" )); assertU(adoc("id", "6", "title", "XXXX XXXX", "str_s1", "z"));
assertU(adoc("id", "7", "title", "AAAA", "str_s1", "a" )); assertU(adoc("id", "7", "title", "AAAA", "str_s1", "a"));
assertU(commit()); assertU(commit());
assertQ("", req(CommonParams.Q, "XXXX", CommonParams.QT, "/elevate", assertQ("", req(CommonParams.Q, "XXXX", CommonParams.QT, "/elevate",
CommonParams.FL, "id, score, [elevated]") CommonParams.FL, "id, score, [elevated]")
,"//*[@numFound='3']" , "//*[@numFound='3']"
,"//result/doc[1]/str[@name='id'][.='1']" , "//result/doc[1]/str[@name='id'][.='1']"
,"//result/doc[2]/str[@name='id'][.='4']" , "//result/doc[2]/str[@name='id'][.='4']"
,"//result/doc[3]/str[@name='id'][.='6']", , "//result/doc[3]/str[@name='id'][.='6']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']", "//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='false']", "//result/doc[2]/bool[@name='[elevated]'][.='false']",
"//result/doc[3]/bool[@name='[elevated]'][.='false']" "//result/doc[3]/bool[@name='[elevated]'][.='false']"
); );
assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate", assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate",
CommonParams.FL, "id, score, [elevated]") CommonParams.FL, "id, score, [elevated]")
@ -231,127 +227,126 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate", assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate",
CommonParams.FL, "id, score, [elev]") CommonParams.FL, "id, score, [elev]")
,"//*[@numFound='1']" , "//*[@numFound='1']"
,"//result/doc[1]/str[@name='id'][.='7']", , "//result/doc[1]/str[@name='id'][.='7']",
"not(//result/doc[1]/bool[@name='[elevated]'][.='false'])", "not(//result/doc[1]/bool[@name='[elevated]'][.='false'])",
"not(//result/doc[1]/bool[@name='[elev]'][.='false'])" // even though we asked for elev, there is no Transformer registered w/ that, so we shouldn't get a result "not(//result/doc[1]/bool[@name='[elev]'][.='false'])" // even though we asked for elev, there is no Transformer registered w/ that, so we shouldn't get a result
); );
} finally { } finally {
delete(); delete();
} }
} }
@Test @Test
public void testSorting() throws Exception public void testSorting() throws Exception {
{
try { try {
init("schema12.xml"); init("schema12.xml");
assertU(adoc("id", "a", "title", "ipod", "str_s1", "a" )); assertU(adoc("id", "a", "title", "ipod", "str_s1", "a"));
assertU(adoc("id", "b", "title", "ipod ipod", "str_s1", "b" )); assertU(adoc("id", "b", "title", "ipod ipod", "str_s1", "b"));
assertU(adoc("id", "c", "title", "ipod ipod ipod", "str_s1", "c" )); assertU(adoc("id", "c", "title", "ipod ipod ipod", "str_s1", "c"));
assertU(adoc("id", "x", "title", "boosted", "str_s1", "x" )); assertU(adoc("id", "x", "title", "boosted", "str_s1", "x"));
assertU(adoc("id", "y", "title", "boosted boosted", "str_s1", "y" )); assertU(adoc("id", "y", "title", "boosted boosted", "str_s1", "y"));
assertU(adoc("id", "z", "title", "boosted boosted boosted", "str_s1", "z" )); assertU(adoc("id", "z", "title", "boosted boosted boosted", "str_s1", "z"));
assertU(commit()); assertU(commit());
String query = "title:ipod"; String query = "title:ipod";
Map<String,String> args = new HashMap<String, String>(); Map<String, String> args = new HashMap<String, String>();
args.put( CommonParams.Q, query ); args.put(CommonParams.Q, query);
args.put( CommonParams.QT, "/elevate" ); args.put(CommonParams.QT, "/elevate");
args.put( CommonParams.FL, "id,score" ); args.put(CommonParams.FL, "id,score");
args.put( "indent", "true" ); args.put("indent", "true");
//args.put( CommonParams.FL, "id,title,score" ); //args.put( CommonParams.FL, "id,title,score" );
SolrQueryRequest req = new LocalSolrQueryRequest( h.getCore(), new MapSolrParams( args) ); SolrQueryRequest req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
IndexReader reader = req.getSearcher().getIndexReader(); IndexReader reader = req.getSearcher().getIndexReader();
QueryElevationComponent booster = (QueryElevationComponent)req.getCore().getSearchComponent( "elevate" ); QueryElevationComponent booster = (QueryElevationComponent) req.getCore().getSearchComponent("elevate");
assertQ("Make sure standard sort works as expected", req assertQ("Make sure standard sort works as expected", req
,"//*[@numFound='3']" , "//*[@numFound='3']"
,"//result/doc[1]/str[@name='id'][.='a']" , "//result/doc[1]/str[@name='id'][.='a']"
,"//result/doc[2]/str[@name='id'][.='b']" , "//result/doc[2]/str[@name='id'][.='b']"
,"//result/doc[3]/str[@name='id'][.='c']" , "//result/doc[3]/str[@name='id'][.='c']"
); );
// Explicitly set what gets boosted // Explicitly set what gets boosted
booster.elevationCache.clear(); booster.elevationCache.clear();
booster.setTopQueryResults( reader, query, new String[] { "x", "y", "z" }, null ); booster.setTopQueryResults(reader, query, new String[]{"x", "y", "z"}, null);
assertQ("All six should make it", req assertQ("All six should make it", req
,"//*[@numFound='6']" , "//*[@numFound='6']"
,"//result/doc[1]/str[@name='id'][.='x']" , "//result/doc[1]/str[@name='id'][.='x']"
,"//result/doc[2]/str[@name='id'][.='y']" , "//result/doc[2]/str[@name='id'][.='y']"
,"//result/doc[3]/str[@name='id'][.='z']" , "//result/doc[3]/str[@name='id'][.='z']"
,"//result/doc[4]/str[@name='id'][.='a']" , "//result/doc[4]/str[@name='id'][.='a']"
,"//result/doc[5]/str[@name='id'][.='b']" , "//result/doc[5]/str[@name='id'][.='b']"
,"//result/doc[6]/str[@name='id'][.='c']" , "//result/doc[6]/str[@name='id'][.='c']"
); );
booster.elevationCache.clear(); booster.elevationCache.clear();
// now switch the order: // now switch the order:
booster.setTopQueryResults( reader, query, new String[] { "a", "x" }, null ); booster.setTopQueryResults(reader, query, new String[]{"a", "x"}, null);
assertQ("All four should make it", req assertQ("All four should make it", req
,"//*[@numFound='4']" , "//*[@numFound='4']"
,"//result/doc[1]/str[@name='id'][.='a']" , "//result/doc[1]/str[@name='id'][.='a']"
,"//result/doc[2]/str[@name='id'][.='x']" , "//result/doc[2]/str[@name='id'][.='x']"
,"//result/doc[3]/str[@name='id'][.='b']" , "//result/doc[3]/str[@name='id'][.='b']"
,"//result/doc[4]/str[@name='id'][.='c']" , "//result/doc[4]/str[@name='id'][.='c']"
); );
// Test reverse sort // Test reverse sort
args.put( CommonParams.SORT, "score asc" ); args.put(CommonParams.SORT, "score asc");
assertQ("All four should make it", req assertQ("All four should make it", req
,"//*[@numFound='4']" , "//*[@numFound='4']"
,"//result/doc[4]/str[@name='id'][.='a']" , "//result/doc[4]/str[@name='id'][.='a']"
,"//result/doc[3]/str[@name='id'][.='x']" , "//result/doc[3]/str[@name='id'][.='x']"
,"//result/doc[2]/str[@name='id'][.='b']" , "//result/doc[2]/str[@name='id'][.='b']"
,"//result/doc[1]/str[@name='id'][.='c']" , "//result/doc[1]/str[@name='id'][.='c']"
); );
// Try normal sort by 'id' // Try normal sort by 'id'
// default 'forceBoost' should be false // default 'forceBoost' should be false
assertEquals( false, booster.forceElevation ); assertEquals(false, booster.forceElevation);
args.put( CommonParams.SORT, "str_s1 asc" ); args.put(CommonParams.SORT, "str_s1 asc");
assertQ( null, req assertQ(null, req
,"//*[@numFound='4']" , "//*[@numFound='4']"
,"//result/doc[1]/str[@name='id'][.='a']" , "//result/doc[1]/str[@name='id'][.='a']"
,"//result/doc[2]/str[@name='id'][.='b']" , "//result/doc[2]/str[@name='id'][.='b']"
,"//result/doc[3]/str[@name='id'][.='c']" , "//result/doc[3]/str[@name='id'][.='c']"
,"//result/doc[4]/str[@name='id'][.='x']" , "//result/doc[4]/str[@name='id'][.='x']"
); );
booster.forceElevation = true; booster.forceElevation = true;
assertQ( null, req assertQ(null, req
,"//*[@numFound='4']" , "//*[@numFound='4']"
,"//result/doc[1]/str[@name='id'][.='a']" , "//result/doc[1]/str[@name='id'][.='a']"
,"//result/doc[2]/str[@name='id'][.='x']" , "//result/doc[2]/str[@name='id'][.='x']"
,"//result/doc[3]/str[@name='id'][.='b']" , "//result/doc[3]/str[@name='id'][.='b']"
,"//result/doc[4]/str[@name='id'][.='c']" , "//result/doc[4]/str[@name='id'][.='c']"
); );
//Test exclusive (not to be confused with exclusion) //Test exclusive (not to be confused with exclusion)
args.put(QueryElevationParams.EXCLUSIVE, "true"); args.put(QueryElevationParams.EXCLUSIVE, "true");
booster.setTopQueryResults( reader, query, new String[] { "x", "a" }, new String[] {} ); booster.setTopQueryResults(reader, query, new String[]{"x", "a"}, new String[]{});
assertQ( null, req assertQ(null, req
,"//*[@numFound='2']" , "//*[@numFound='2']"
,"//result/doc[1]/str[@name='id'][.='x']" , "//result/doc[1]/str[@name='id'][.='x']"
,"//result/doc[2]/str[@name='id'][.='a']" , "//result/doc[2]/str[@name='id'][.='a']"
); );
// Test exclusion // Test exclusion
booster.elevationCache.clear(); booster.elevationCache.clear();
args.remove( CommonParams.SORT ); args.remove(CommonParams.SORT);
args.remove( QueryElevationParams.EXCLUSIVE); args.remove(QueryElevationParams.EXCLUSIVE);
booster.setTopQueryResults( reader, query, new String[] { "x" }, new String[] { "a" } ); booster.setTopQueryResults(reader, query, new String[]{"x"}, new String[]{"a"});
assertQ( null, req assertQ(null, req
,"//*[@numFound='3']" , "//*[@numFound='3']"
,"//result/doc[1]/str[@name='id'][.='x']" , "//result/doc[1]/str[@name='id'][.='x']"
,"//result/doc[2]/str[@name='id'][.='b']" , "//result/doc[2]/str[@name='id'][.='b']"
,"//result/doc[3]/str[@name='id'][.='c']" , "//result/doc[3]/str[@name='id'][.='c']"
); );
req.close(); req.close();
@ -359,57 +354,55 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
delete(); delete();
} }
} }
// write a test file to boost some docs // write a test file to boost some docs
private void writeFile( File file, String query, String ... ids ) throws Exception private void writeFile(File file, String query, String... ids) throws Exception {
{ PrintWriter out = new PrintWriter(new FileOutputStream(file));
PrintWriter out = new PrintWriter( new FileOutputStream( file ) ); out.println("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>");
out.println( "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>" ); out.println("<elevate>");
out.println( "<elevate>" ); out.println("<query text=\"" + query + "\">");
out.println( "<query text=\""+query+"\">" ); for (String id : ids) {
for( String id : ids ) { out.println(" <doc id=\"" + id + "\"/>");
out.println( " <doc id=\""+id+"\"/>" );
} }
out.println( "</query>" ); out.println("</query>");
out.println( "</elevate>" ); out.println("</elevate>");
out.flush(); out.flush();
out.close(); out.close();
log.info( "OUT:"+file.getAbsolutePath() ); log.info("OUT:" + file.getAbsolutePath());
} }
@Test @Test
public void testElevationReloading() throws Exception public void testElevationReloading() throws Exception {
{
try { try {
init("schema12.xml"); init("schema12.xml");
String testfile = "data-elevation.xml"; String testfile = "data-elevation.xml";
File f = new File( h.getCore().getDataDir(), testfile ); File f = new File(h.getCore().getDataDir(), testfile);
writeFile( f, "aaa", "A" ); writeFile(f, "aaa", "A");
QueryElevationComponent comp = (QueryElevationComponent)h.getCore().getSearchComponent("elevate"); QueryElevationComponent comp = (QueryElevationComponent) h.getCore().getSearchComponent("elevate");
NamedList<String> args = new NamedList<String>(); NamedList<String> args = new NamedList<String>();
args.add( QueryElevationComponent.CONFIG_FILE, testfile ); args.add(QueryElevationComponent.CONFIG_FILE, testfile);
comp.init( args ); comp.init(args);
comp.inform( h.getCore() ); comp.inform(h.getCore());
SolrQueryRequest req = req(); SolrQueryRequest req = req();
IndexReader reader = req.getSearcher().getIndexReader(); IndexReader reader = req.getSearcher().getIndexReader();
Map<String, ElevationObj> map = comp.getElevationMap(reader, h.getCore()); Map<String, ElevationObj> map = comp.getElevationMap(reader, h.getCore());
assertTrue( map.get( "aaa" ).priority.containsKey( new BytesRef("A") ) ); assertTrue(map.get("aaa").priority.containsKey(new BytesRef("A")));
assertNull( map.get( "bbb" ) ); assertNull(map.get("bbb"));
req.close(); req.close();
// now change the file // now change the file
writeFile( f, "bbb", "B" ); writeFile(f, "bbb", "B");
assertU(adoc("id", "10000")); // will get same reader if no index change assertU(adoc("id", "10000")); // will get same reader if no index change
assertU(commit()); assertU(commit());
req = req(); req = req();
reader = req.getSearcher().getIndexReader(); reader = req.getSearcher().getIndexReader();
map = comp.getElevationMap(reader, h.getCore()); map = comp.getElevationMap(reader, h.getCore());
assertNull( map.get( "aaa" ) ); assertNull(map.get("aaa"));
assertTrue( map.get( "bbb" ).priority.containsKey( new BytesRef("B") ) ); assertTrue(map.get("bbb").priority.containsKey(new BytesRef("B")));
req.close(); req.close();
} finally { } finally {
delete(); delete();