Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-303 Distributed Search over HTTP 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@631357 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2008-02-26 19:47:07 +00:00
parent 0c2e3ebdd4
commit cc61bb647b
30 changed files with 2799 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGES.txt
View File

@ -199,6 +199,15 @@ New Features
with "304 Not Modified" when appropriate. New options have been added
to solrconfig.xml to influence this behavior.
(Thomas Peuss via hossman)
40. SOLR-303: Distributed Search over HTTP. Specification of shards
argument causes Solr to query those shards and merge the results
into a single response. Querying, field faceting (sorted only),
query faceting, highlighting, and debug information are supported
in distributed mode.
(Sharad Agarwal, Patrick O'Leary, Sabyasachi Dalal, Stu Hood,
ryan, yonik)
Changes in runtime behavior

4
build.xml
View File

@ -235,9 +235,10 @@
depends="compile-common,init-forrest-entities">
<solr-javac destdir="${dest}/core"
classpathref="compile.classpath">
classpathref="compile.classpath.solrj-embedded">
<src path="${src}/java" />
<src path="${src}/webapp/src" />
<src path="client/java/solrj/src" />
<exclude name="org/apache/solr/common/**" />
</solr-javac>
</target>
@ -628,6 +629,7 @@
<include name="${fullnamever}.jar" />
<include name="${fullname}-common-${version}.jar" />
</lib>
<lib dir="client/java/solrj/lib"/>
<fileset dir="${src}/webapp/resources" />
<metainf dir="${basedir}" includes="LICENSE.txt,NOTICE.txt"/>
</war>

20
client/java/solrj/src/org/apache/solr/client/solrj/response/QueryResponse.java
View File

@ -51,7 +51,6 @@ public class QueryResponse extends SolrResponseBase
// Debug Info
private Map<String,Object> _debugMap = null;
private Map<String,Integer> _docIdMap = null;
private Map<String,String> _explainMap = null;
public QueryResponse( NamedList<Object> res )
@ -81,9 +80,7 @@ public class QueryResponse extends SolrResponseBase
}
}
}
private static final String DKEY = ",internal_docid=";
private void extractDebugInfo( NamedList<Object> debug )
{
_debugMap = new LinkedHashMap<String, Object>(); // keep the order
@ -93,20 +90,11 @@ public class QueryResponse extends SolrResponseBase
// Parse out interisting bits from the debug info
_explainMap = new HashMap<String, String>();
_docIdMap = new HashMap<String, Integer>();
NamedList<String> explain = (NamedList<String>)_debugMap.get( "explain" );
if( explain != null ) {
for( Map.Entry<String, String> info : explain ) {
String key = info.getKey();
int idx0 = key.indexOf( '=' )+1;
int idx1 = info.getKey().indexOf( DKEY );
int idx2 = idx1 + DKEY.length();
String id = key.substring( idx0, idx1 );
String docID = key.substring( idx2 );
_explainMap.put( id, info.getValue() );
_docIdMap.put( id, Integer.valueOf( docID ) );
_explainMap.put( key, info.getValue() );
}
}
}
@ -182,10 +170,6 @@ public class QueryResponse extends SolrResponseBase
return _debugMap;
}
public Map<String, Integer> getDocIdMap() {
return _docIdMap;
}
public Map<String, String> getExplainMap() {
return _explainMap;
}

2
client/java/solrj/src/org/apache/solr/client/solrj/response/SolrResponseBase.java
View File

@ -25,7 +25,7 @@ import org.apache.solr.common.util.NamedList;
* @version $Id$
* @since solr 1.3
*/
public abstract class SolrResponseBase extends SolrResponse
public class SolrResponseBase extends SolrResponse
{
private long elapsedTime = -1;
private NamedList<Object> response = null;

21
src/java/org/apache/solr/common/util/StrUtils.java
View File

@ -118,6 +118,27 @@ public class StrUtils {
return lst;
}
/** Creates a backslash escaped string, joining all the items. */
public static String join(List<String> items, char separator) {
StringBuilder sb = new StringBuilder(items.size() << 3);
boolean first=true;
for (String item : items) {
if (first) {
first = false;
} else {
sb.append(separator);
}
for (int i=0; i<item.length(); i++) {
char ch = item.charAt(i);
if (ch=='\\' || ch == separator) {
sb.append('\\');
}
sb.append(ch);
}
}
return sb.toString();
}
public static List<String> splitWS(String s, boolean decode) {

2
src/java/org/apache/solr/handler/MoreLikeThisHandler.java
View File

@ -179,7 +179,7 @@ public class MoreLikeThisHandler extends RequestHandlerBase
rsp.add( "facet_counts", null );
}
else {
SimpleFacets f = new SimpleFacets(searcher, mltDocs.docSet, params );
SimpleFacets f = new SimpleFacets(req, mltDocs.docSet, params );
rsp.add( "facet_counts", f.getFacetCounts() );
}
}

4
src/java/org/apache/solr/handler/RequestHandlerBase.java
View File

@ -27,6 +27,7 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.util.SolrPluginUtils;
import org.apache.lucene.queryParser.ParseException;
import java.net.URL;
@ -117,6 +118,9 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
handleRequestBody( req, rsp );
} catch (Exception e) {
SolrException.log(SolrCore.log,e);
if (e instanceof ParseException) {
e = new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
rsp.setException(e);
numErrors++;
}

5
src/java/org/apache/solr/handler/StandardRequestHandler.java
View File

@ -17,11 +17,12 @@
package org.apache.solr.handler;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.component.*;
import java.net.MalformedURLException;
import java.net.URL;
import org.apache.solr.handler.component.SearchHandler;
/**
* @version $Id$
*

177
src/java/org/apache/solr/handler/component/DebugComponent.java
View File

@ -18,21 +18,20 @@
package org.apache.solr.handler.component;
import static org.apache.solr.common.params.CommonParams.FQ;
import org.apache.solr.common.params.HighlightParams;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.*;
import org.apache.lucene.search.Query;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.search.QueryParsing;
import org.apache.solr.util.SolrPluginUtils;
/**
* TODO!
* Adds debugging information to a request.
*
* @version $Id$
* @since solr 1.3
@ -42,48 +41,182 @@ public class DebugComponent extends SearchComponent
public static final String COMPONENT_NAME = "debug";
@Override
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
public void prepare(ResponseBuilder rb) throws IOException
{
}
@SuppressWarnings("unchecked")
@Override
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
public void process(ResponseBuilder rb) throws IOException
{
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
if( builder.isDebug() ) {
NamedList stdinfo = SolrPluginUtils.doStandardDebug( req,
builder.getQueryString(), builder.getQuery(), builder.getResults().docList);
if( rb.isDebug() ) {
NamedList stdinfo = SolrPluginUtils.doStandardDebug( rb.req,
rb.getQueryString(), rb.getQuery(), rb.getResults().docList);
NamedList info = builder.getDebugInfo();
NamedList info = rb.getDebugInfo();
if( info == null ) {
builder.setDebugInfo( stdinfo );
rb.setDebugInfo( stdinfo );
info = stdinfo;
}
else {
info.addAll( stdinfo );
}
if (builder.getQparser() != null) {
builder.getQparser().addDebugInfo(builder.getDebugInfo());
if (rb.getQparser() != null) {
rb.getQparser().addDebugInfo(rb.getDebugInfo());
}
if (null != builder.getDebugInfo() ) {
if (null != builder.getFilters() ) {
info.add("filter_queries",req.getParams().getParams(FQ));
List<String> fqs = new ArrayList<String>(builder.getFilters().size());
for (Query fq : builder.getFilters()) {
fqs.add(QueryParsing.toString(fq, req.getSchema()));
if (null != rb.getDebugInfo() ) {
if (null != rb.getFilters() ) {
info.add("filter_queries",rb.req.getParams().getParams(FQ));
List<String> fqs = new ArrayList<String>(rb.getFilters().size());
for (Query fq : rb.getFilters()) {
fqs.add(QueryParsing.toString(fq, rb.req.getSchema()));
}
info.add("parsed_filter_queries",fqs);
}
// Add this directly here?
rsp.add("debug", builder.getDebugInfo() );
rb.rsp.add("debug", rb.getDebugInfo() );
}
}
}
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
if (!rb.isDebug()) return;
// Turn on debug to get explain only only when retrieving fields
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
sreq.purpose |= ShardRequest.PURPOSE_GET_DEBUG;
sreq.params.set("debugQuery", "true");
} else {
sreq.params.set("debugQuery", "false");
}
}
@Override
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
}
private Set<String> excludeSet = new HashSet<String>(Arrays.asList("explain"));
@Override
public void finishStage(ResponseBuilder rb) {
if (rb.isDebug() && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
NamedList info = null;
NamedList explain = new SimpleOrderedMap();
Object[] arr = new Object[rb.resultIds.size() * 2];
for (ShardRequest sreq : rb.finished) {
if ((sreq.purpose & ShardRequest.PURPOSE_GET_DEBUG) == 0) continue;
for (ShardResponse srsp : sreq.responses) {
NamedList sdebug = (NamedList)srsp.rsp.getResponse().get("debug");
info = (NamedList)merge(sdebug, info, excludeSet);
NamedList sexplain = (NamedList)sdebug.get("explain");
for (int i=0; i<sexplain.size(); i++) {
String id = sexplain.getName(i);
// TODO: lookup won't work for non-string ids... String vs Float
ShardDoc sdoc = rb.resultIds.get(id);
int idx = sdoc.positionInResponse;
arr[idx<<1] = id;
arr[(idx<<1)+1] = sexplain.getVal(i);
}
}
}
explain = HighlightComponent.removeNulls(new NamedList(Arrays.asList(arr)));
int idx = info.indexOf("explain",0);
if (idx>=0) {
info.setVal(idx, explain);
} else {
info.add("explain", explain);
}
rb.setDebugInfo(info);
rb.rsp.add("debug", rb.getDebugInfo() );
}
}
Object merge(Object source, Object dest, Set<String> exclude) {
if (source == null) return dest;
if (dest == null) {
if (source instanceof NamedList) {
dest = source instanceof SimpleOrderedMap ? new SimpleOrderedMap() : new NamedList();
} else {
return source;
}
} else {
if (dest instanceof Collection) {
if (source instanceof Collection) {
((Collection)dest).addAll((Collection)source);
} else {
((Collection)dest).add(source);
}
return dest;
} else if (source instanceof Number) {
if (dest instanceof Number) {
if (source instanceof Double || dest instanceof Double) {
return ((Number)source).doubleValue() + ((Number)dest).doubleValue();
}
return ((Number)source).longValue() + ((Number)dest).longValue();
}
// fall through
} else if (source instanceof String) {
if (source.equals(dest)) {
return dest;
}
// fall through
}
}
if (source instanceof NamedList && dest instanceof NamedList) {
NamedList tmp = new NamedList();
NamedList sl = (NamedList)source;
NamedList dl = (NamedList)dest;
for (int i=0; i<sl.size(); i++) {
String skey = sl.getName(i);
if (exclude != null && exclude.contains(skey)) continue;
Object sval = sl.getVal(i);
int didx = -1;
// optimize case where elements are in same position
if (i < dl.size()) {
String dkey = dl.getName(i);
if (skey == dkey || (skey!=null && skey.equals(dkey))) {
didx = i;
}
}
if (didx == -1) {
didx = dl.indexOf(skey, 0);
}
if (didx == -1) {
tmp.add(skey, merge(sval, null, null));
} else {
dl.setVal(didx, merge(sval, dl.getVal(didx), null));
}
}
dl.addAll(tmp);
return dl;
}
// merge unlike elements in a list
List t = new ArrayList();
t.add(dest);
t.add(source);
return t;
}
/////////////////////////////////////////////
/// SolrInfoMBean

511
src/java/org/apache/solr/handler/component/FacetComponent.java
View File

@ -19,12 +19,19 @@ package org.apache.solr.handler.component;
import java.io.IOException;
import java.net.URL;
import java.util.*;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.SolrException;
import org.apache.solr.request.SimpleFacets;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.util.OpenBitSet;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.QueryParsing;
import org.apache.lucene.queryParser.ParseException;
/**
* TODO!
@ -37,34 +44,336 @@ public class FacetComponent extends SearchComponent
public static final String COMPONENT_NAME = "facet";
@Override
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
public void prepare(ResponseBuilder rb) throws IOException
{
SolrParams params = req.getParams();
if (params.getBool(FacetParams.FACET,false)) {
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
builder.setNeedDocSet( true );
if (rb.req.getParams().getBool(FacetParams.FACET,false)) {
rb.setNeedDocSet( true );
rb.doFacets = true;
}
}
/**
* Actually run the query
* @param rb
*/
@Override
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
public void process(ResponseBuilder rb) throws IOException
{
SolrParams params = req.getParams();
if (params.getBool(FacetParams.FACET,false)) {
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
SimpleFacets f = new SimpleFacets(req.getSearcher(),
builder.getResults().docSet,
params );
if (rb.doFacets) {
SolrParams params = rb.req.getParams();
SimpleFacets f = new SimpleFacets(rb.req,
rb.getResults().docSet,
params );
// TODO ???? add this directly to the response?
rsp.add( "facet_counts", f.getFacetCounts() );
// TODO ???? add this directly to the response, or to the builder?
rb.rsp.add( "facet_counts", f.getFacetCounts() );
}
}
@Override
public int distributedProcess(ResponseBuilder rb) throws IOException {
if (!rb.doFacets) {
return ResponseBuilder.STAGE_DONE;
}
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
// overlap facet refinement requests (those shards that we need a count for
// particular facet values from), where possible, with
// the requests to get fields (because we know that is the
// only other required phase).
// We do this in distributedProcess so we can look at all of the
// requests in the outgoing queue at once.
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
List<String> fqueries = rb._facetInfo._toRefine[shardNum];
if (fqueries == null || fqueries.size()==0) continue;
String shard = rb.shards[shardNum];
ShardRequest refine = null;
boolean newRequest = false;
// try to find a request that is already going out to that shard.
// If nshards becomes to great, we way want to move to hashing for better
// scalability.
for (ShardRequest sreq : rb.outgoing) {
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS)!=0
&& sreq.shards != null & sreq.shards.length==1
&& sreq.shards[0].equals(shard))
{
refine = sreq;
break;
}
}
if (refine == null) {
// we didn't find any other suitable requests going out to that shard, so
// create one ourselves.
newRequest = true;
refine = new ShardRequest();
refine.shards = new String[]{rb.shards[shardNum]};
refine.params = new ModifiableSolrParams(rb.req.getParams());
// don't request any documents
refine.params.remove("start");
refine.params.set("rows","0");
}
refine.purpose |= ShardRequest.PURPOSE_REFINE_FACETS;
refine.params.set(FacetParams.FACET,"true");
refine.params.remove(FacetParams.FACET_FIELD);
// TODO: perhaps create a more compact facet.terms method?
refine.params.set(FacetParams.FACET_QUERY, fqueries.toArray(new String[fqueries.size()]));
if (newRequest) {
rb.addRequest(this, refine);
}
}
}
return ResponseBuilder.STAGE_DONE;
}
@Override
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
if (!rb.doFacets) return;
if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
sreq.purpose |= ShardRequest.PURPOSE_GET_FACETS;
FacetInfo fi = rb._facetInfo;
if (fi == null) {
rb._facetInfo = fi = new FacetInfo();
fi.parse(rb.req.getParams(), rb);
// should already be true...
// sreq.params.set(FacetParams.FACET, "true");
}
sreq.params.remove(FacetParams.FACET_MINCOUNT);
sreq.params.remove(FacetParams.FACET_OFFSET);
sreq.params.remove(FacetParams.FACET_LIMIT);
for (DistribFieldFacet dff : fi.topFacets.values()) {
String paramStart = "f." + dff.field + '.';
sreq.params.remove(paramStart + FacetParams.FACET_MINCOUNT);
sreq.params.remove(paramStart + FacetParams.FACET_OFFSET);
// set the initial limit higher in increase accuracy
dff.initialLimit = dff.offset + dff.limit;
dff.initialLimit = (int)(dff.initialLimit * 1.5) + 10;
// Uncomment the following line when testing to supress over-requesting facets and
// thus cause more facet refinement queries.
// dff.initialLimit = dff.offset + dff.limit;
sreq.params.set(paramStart + FacetParams.FACET_LIMIT, dff.initialLimit);
}
} else {
// turn off faceting on other requests
sreq.params.set(FacetParams.FACET, "false");
// we could optionally remove faceting params
}
}
@Override
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
if (!rb.doFacets) return;
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FACETS)!=0) {
countFacets(rb, sreq);
} else if ((sreq.purpose & ShardRequest.PURPOSE_REFINE_FACETS)!=0) {
refineFacets(rb, sreq);
}
}
private void countFacets(ResponseBuilder rb, ShardRequest sreq) {
FacetInfo fi = rb._facetInfo;
for (ShardResponse srsp: sreq.responses) {
int shardNum = rb.getShardNum(srsp.shard);
NamedList facet_counts = (NamedList)srsp.rsp.getResponse().get("facet_counts");
// handle facet queries
NamedList facet_queries = (NamedList)facet_counts.get("facet_queries");
if (facet_queries != null) {
for (int i=0; i<facet_queries.size(); i++) {
String facet_q = (String)facet_queries.getName(i);
long count = ((Number)facet_queries.getVal(i)).longValue();
Long prevCount = fi.queryFacets.get(facet_q);
if (prevCount != null) count += prevCount;
fi.queryFacets.put(facet_q, count);
}
}
// step through each facet.field, adding results from this shard
NamedList facet_fields = (NamedList)facet_counts.get("facet_fields");
for (DistribFieldFacet dff : fi.topFacets.values()) {
dff.add(shardNum, (NamedList)facet_fields.get(dff.field), dff.initialLimit);
}
}
//
// This code currently assumes that there will be only a single
// request ((with responses from all shards) sent out to get facets...
// otherwise we would need to wait until all facet responses were received.
//
// list of queries to send each shard
List<String>[] toRefine = new List[rb.shards.length];
fi._toRefine = toRefine;
for (int i=0; i<toRefine.length; i++) {
toRefine[i] = new ArrayList<String>();
}
for (DistribFieldFacet dff : fi.topFacets.values()) {
ShardFacetCount[] counts = dff.getSorted();
int ntop = Math.min(counts.length, dff.offset + dff.limit);
long smallestCount = counts[ntop-1].count;
for (int i=0; i<counts.length; i++) {
ShardFacetCount sfc = counts[i];
String query = null;
boolean needRefinement = false;
if (i<ntop) {
// automatically flag the top values for refinement
needRefinement = true;
} else {
// calculate the maximum value that this term may have
// and if it is >= smallestCount, then flag for refinement
long maxCount = sfc.count;
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
OpenBitSet obs = dff.counted[shardNum];
if (!obs.get(sfc.termNum)) {
// if missing from this shard, add the max it could be
maxCount += dff.maxPossible(sfc,shardNum);
}
}
if (maxCount >= smallestCount) {
// TODO: on a tie, we could check the term values
needRefinement = true;
}
}
if (needRefinement) {
// add a query for each shard missing the term that needs refinement
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
OpenBitSet obs = dff.counted[shardNum];
if (!obs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
dff.needRefinements = true;
if (query==null) query = dff.makeQuery(sfc);
toRefine[shardNum].add(query);
}
}
}
}
}
}
private void refineFacets(ResponseBuilder rb, ShardRequest sreq) {
FacetInfo fi = rb._facetInfo;
for (ShardResponse srsp: sreq.responses) {
int shardNum = rb.getShardNum(srsp.shard);
NamedList facet_counts = (NamedList)srsp.rsp.getResponse().get("facet_counts");
NamedList facet_queries = (NamedList)facet_counts.get("facet_queries");
// These are single term queries used to fill in missing counts
// for facet.field queries
for (int i=0; i<facet_queries.size(); i++) {
try {
String facet_q = (String)facet_queries.getName(i);
long count = ((Number)facet_queries.getVal(i)).longValue();
// expect <!field f=field>value style params
SolrParams qparams = QueryParsing.getLocalParams(facet_q,null);
String field = qparams.get(QueryParsing.F);
String val = qparams.get(QueryParsing.V);
// Find the right field.facet for this field
DistribFieldFacet dff = fi.topFacets.get(field);
// Find the right constraint count for this value
ShardFacetCount sfc = dff.counts.get(val);
// TODO REMOVE
System.out.println("Got " + facet_q + " , refining count: " + sfc + " += " + count);
sfc.count += count;
} catch (ParseException e) {
// shouldn't happen, so fail for now rather than covering it up
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
}
}
@Override
public void finishStage(ResponseBuilder rb) {
if (!rb.doFacets || rb.stage != ResponseBuilder.STAGE_GET_FIELDS) return;
// wait until STAGE_GET_FIELDS
// so that "result" is already stored in the response (for aesthetics)
FacetInfo fi = rb._facetInfo;
NamedList facet_counts = new NamedList();
NamedList facet_queries = new NamedList();
facet_counts.add("facet_queries",facet_queries);
for (Map.Entry<String,Long> entry : fi.queryFacets.entrySet()) {
facet_queries.add(entry.getKey(), num(entry.getValue()));
}
NamedList facet_fields = new NamedList();
facet_counts.add("facet_fields", facet_fields);
for (DistribFieldFacet dff : fi.topFacets.values()) {
SimpleOrderedMap fieldCounts = new SimpleOrderedMap();
facet_fields.add(dff.field, fieldCounts);
ShardFacetCount[] counts = dff.countSorted;
if (dff.needRefinements) {
counts = dff.getSorted();
}
int end = Math.min(dff.offset + dff.limit, counts.length);
for (int i=dff.offset; i<end; i++) {
if (counts[i].count < dff.minCount) break;
fieldCounts.add(counts[i].name, num(counts[i].count));
}
if (dff.missing) {
fieldCounts.add(null, num(dff.missingCount));
}
}
// TODO: list facets (sorted by natural order)
// TODO: facet dates
facet_counts.add("facet_dates", new NamedList());
rb.rsp.add("facet_counts", facet_counts);
rb._facetInfo = null; // could be big, so release asap
}
// use <int> tags for smaller facet counts (better back compatibility)
private Number num(long val) {
if (val < Integer.MAX_VALUE) return (int)val;
else return val;
}
private Number num(Long val) {
if (val.longValue() < Integer.MAX_VALUE) return val.intValue();
else return val;
}
/////////////////////////////////////////////
/// SolrInfoMBean
////////////////////////////////////////////
@ -94,3 +403,171 @@ public class FacetComponent extends SearchComponent
return null;
}
}
class FacetInfo {
List<String>[] _toRefine;
void parse(SolrParams params, ResponseBuilder rb) {
queryFacets = new LinkedHashMap<String,Long>();
topFacets = new LinkedHashMap<String,DistribFieldFacet>();
listFacets = new LinkedHashMap<String,DistribFieldFacet>();
String[] facetQs = params.getParams(FacetParams.FACET_QUERY);
if (facetQs != null) {
for (String query : facetQs) {
queryFacets.put(query,0L);
}
}
String[] facetFs = params.getParams(FacetParams.FACET_FIELD);
if (facetFs != null) {
for (String field : facetFs) {
DistribFieldFacet ff = new DistribFieldFacet(rb, field);
ff.fillParams(params, field);
if (ff.sort) {
topFacets.put(field, ff);
} else {
listFacets.put(field, ff);
}
}
}
}
LinkedHashMap<String,Long> queryFacets;
LinkedHashMap<String,DistribFieldFacet> topFacets; // field facets that order by constraint count (sort=true)
LinkedHashMap<String,DistribFieldFacet> listFacets; // field facets that list values in term order
}
class FieldFacet {
String field;
int offset;
int limit;
int minCount;
boolean sort;
boolean missing;
String prefix;
long missingCount;
void fillParams(SolrParams params, String field) {
this.field = field;
this.offset = params.getFieldInt(field, FacetParams.FACET_OFFSET, 0);
this.limit = params.getFieldInt(field, FacetParams.FACET_LIMIT, 100);
Integer mincount = params.getFieldInt(field, FacetParams.FACET_MINCOUNT);
if (mincount==null) {
Boolean zeros = params.getFieldBool(field, FacetParams.FACET_ZEROS);
// mincount = (zeros!=null && zeros) ? 0 : 1;
mincount = (zeros!=null && !zeros) ? 1 : 0;
// current default is to include zeros.
}
this.minCount = mincount;
this.missing = params.getFieldBool(field, FacetParams.FACET_MISSING, false);
// default to sorting if there is a limit.
this.sort = params.getFieldBool(field, FacetParams.FACET_SORT, limit>0);
this.prefix = params.getFieldParam(field,FacetParams.FACET_PREFIX);
}
}
class DistribFieldFacet extends FieldFacet {
SchemaField sf;
// the max possible count for a term appearing on no list
long missingMaxPossible;
// the max possible count for a missing term for each shard (indexed by shardNum)
long[] missingMax;
OpenBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
HashMap<String,ShardFacetCount> counts = new HashMap<String,ShardFacetCount>(128);
int termNum;
String queryPrefix;
int initialLimit; // how many terms requested in first phase
boolean needRefinements;
ShardFacetCount[] countSorted;
DistribFieldFacet(ResponseBuilder rb, String field) {
sf = rb.req.getSchema().getField(field);
missingMax = new long[rb.shards.length];
counted = new OpenBitSet[rb.shards.length];
queryPrefix = "<!field f=" + field + '>';
}
void add(int shardNum, NamedList shardCounts, int numRequested) {
int sz = shardCounts.size();
int numReceived = sz;
OpenBitSet terms = new OpenBitSet(termNum+sz);
long last = 0;
for (int i=0; i<sz; i++) {
String name = shardCounts.getName(i);
long count = ((Number)shardCounts.getVal(i)).longValue();
if (name == null) {
missingCount += count;
numReceived--;
} else {
ShardFacetCount sfc = counts.get(name);
if (sfc == null) {
sfc = new ShardFacetCount();
sfc.name = name;
sfc.termNum = termNum++;
counts.put(name, sfc);
}
sfc.count += count;
terms.fastSet(sfc.termNum);
last = count;
}
}
// the largest possible missing term is 0 if we received less
// than the number requested (provided mincount==0 like it should be for
// a shard request)
if (numRequested !=0 && numReceived < numRequested) {
last = 0;
}
missingMaxPossible += last;
missingMax[shardNum] = last;
counted[shardNum] = terms;
}
ShardFacetCount[] getSorted() {
ShardFacetCount[] arr = counts.values().toArray(new ShardFacetCount[counts.size()]);
Arrays.sort(arr, new Comparator<ShardFacetCount>() {
public int compare(ShardFacetCount o1, ShardFacetCount o2) {
if (o2.count < o1.count) return -1;
else if (o1.count < o2.count) return 1;
// TODO: handle tiebreaks for types other than strings
return o1.name.compareTo(o2.name);
}
});
countSorted = arr;
return arr;
}
String makeQuery(ShardFacetCount sfc) {
return queryPrefix + sfc.name;
}
// returns the max possible value this ShardFacetCount could have for this shard
// (assumes the shard did not report a count for this value)
long maxPossible(ShardFacetCount sfc, int shardNum) {
return missingMax[shardNum];
// TODO: could store the last term in the shard to tell if this term
// comes before or after it. If it comes before, we could subtract 1
}
}
class ShardFacetCount {
String name;
long count;
int termNum; // term number starting at 0 (used in bit arrays)
public String toString() {
return "{term="+name+",termNum="+termNum+",count="+count+"}";
}
}

100
src/java/org/apache/solr/handler/component/HighlightComponent.java
View File

@ -19,14 +19,16 @@ package org.apache.solr.handler.component;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.HighlightParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.highlight.SolrHighlighter;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import java.io.IOException;
import java.net.URL;
import java.util.Arrays;
/**
* TODO!
@ -39,51 +41,113 @@ public class HighlightComponent extends SearchComponent
public static final String COMPONENT_NAME = "highlight";
@Override
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp)
public void prepare(ResponseBuilder rb) throws IOException
{
SolrHighlighter highlighter = rb.req.getCore().getHighlighter();
rb.doHighlights = highlighter.isHighlightingEnabled(rb.req.getParams());
}
@Override
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
SolrHighlighter highlighter = req.getCore().getHighlighter();
if (highlighter.isHighlightingEnabled(req.getParams())) {
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
public void process(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
if (rb.doHighlights) {
SolrHighlighter highlighter = req.getCore().getHighlighter();
SolrParams params = req.getParams();
String[] defaultHighlightFields; //TODO: get from builder by default?
if (builder.getQparser() != null) {
defaultHighlightFields = builder.getQparser().getDefaultHighlightFields();
if (rb.getQparser() != null) {
defaultHighlightFields = rb.getQparser().getDefaultHighlightFields();
} else {
defaultHighlightFields = params.getParams(CommonParams.DF);
}
if(builder.getHighlightQuery()==null) {
if (builder.getQparser() != null) {
if(rb.getHighlightQuery()==null) {
if (rb.getQparser() != null) {
try {
builder.setHighlightQuery( builder.getQparser().getHighlightQuery() );
rb.setHighlightQuery( rb.getQparser().getHighlightQuery() );
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
} else {
builder.setHighlightQuery( builder.getQuery() );
rb.setHighlightQuery( rb.getQuery() );
}
}
NamedList sumData = highlighter.doHighlighting(
builder.getResults().docList,
builder.getHighlightQuery().rewrite(req.getSearcher().getReader()),
rb.getResults().docList,
rb.getHighlightQuery().rewrite(req.getSearcher().getReader()),
req, defaultHighlightFields );
if(sumData != null) {
// TODO ???? add this directly to the response?
rsp.add("highlighting", sumData);
rb.rsp.add("highlighting", sumData);
}
}
}
/////////////////////////////////////////////
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
if (!rb.doHighlights) return;
// Turn on highlighting only only when retrieving fields
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
sreq.purpose |= ShardRequest.PURPOSE_GET_HIGHLIGHTS;
// should already be true...
sreq.params.set(HighlightParams.HIGHLIGHT, "true");
} else {
sreq.params.set(HighlightParams.HIGHLIGHT, "false");
}
}
@Override
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
}
@Override
public void finishStage(ResponseBuilder rb) {
if (rb.doHighlights && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
NamedList hlResult = new SimpleOrderedMap();
Object[] arr = new Object[rb.resultIds.size() * 2];
// TODO: make a generic routine to do automatic merging of id keyed data
for (ShardRequest sreq : rb.finished) {
if ((sreq.purpose & ShardRequest.PURPOSE_GET_HIGHLIGHTS) == 0) continue;
for (ShardResponse srsp : sreq.responses) {
NamedList hl = (NamedList)srsp.rsp.getResponse().get("highlighting");
for (int i=0; i<hl.size(); i++) {
String id = hl.getName(i);
ShardDoc sdoc = rb.resultIds.get(id);
int idx = sdoc.positionInResponse;
arr[idx<<1] = id;
arr[(idx<<1)+1] = hl.getVal(i);
}
}
}
// remove nulls in case not all docs were able to be retrieved
rb.rsp.add("highlighting", removeNulls(new NamedList(Arrays.asList(arr))));
}
}
static NamedList removeNulls(NamedList nl) {
for (int i=0; i<nl.size(); i++) {
if (nl.getName(i)==null) {
NamedList newList = new NamedList();
for (int j=0; j<nl.size(); j++) {
String n = nl.getName(j);
if (n != null) {
newList.add(n, nl.getVal(j));
}
}
return newList;
}
}
return nl;
}
////////////////////////////////////////////
/// SolrInfoMBean
////////////////////////////////////////////

15
src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java
View File

@ -24,8 +24,6 @@ import org.apache.solr.common.params.MoreLikeThisParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.MoreLikeThisHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
@ -40,28 +38,27 @@ public class MoreLikeThisComponent extends SearchComponent
public static final String COMPONENT_NAME = "mlt";
@Override
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
public void prepare(ResponseBuilder rb) throws IOException
{
}
@Override
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
public void process(ResponseBuilder rb) throws IOException
{
SolrParams p = req.getParams();
SolrParams p = rb.req.getParams();
if( p.getBool( MoreLikeThisParams.MLT, false ) ) {
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
SolrIndexSearcher searcher = req.getSearcher();
SolrIndexSearcher searcher = rb.req.getSearcher();
MoreLikeThisHandler.MoreLikeThisHelper mlt
= new MoreLikeThisHandler.MoreLikeThisHelper( p, searcher );
int mltcount = p.getInt( MoreLikeThisParams.DOC_COUNT, 5 );
NamedList<DocList> sim = mlt.getMoreLikeThese(
builder.getResults().docList, mltcount, builder.getFieldFlags() );
rb.getResults().docList, mltcount, rb.getFieldFlags() );
// TODO ???? add this directly to the response?
rsp.add( "moreLikeThis", sim );
rb.rsp.add( "moreLikeThis", sim );
}
}

760
src/java/org/apache/solr/handler/component/QueryComponent.java
View File

@ -17,19 +17,32 @@
package org.apache.solr.handler.component;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.*;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.search.*;
import org.apache.solr.util.SolrPluginUtils;
import java.io.IOException;
import java.io.Reader;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.*;
import java.text.Collator;
/**
* TODO!
@ -42,9 +55,10 @@ public class QueryComponent extends SearchComponent
public static final String COMPONENT_NAME = "query";
@Override
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, ParseException
public void prepare(ResponseBuilder rb) throws IOException
{
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
// Set field flags
@ -53,32 +67,43 @@ public class QueryComponent extends SearchComponent
if (fl != null) {
fieldFlags |= SolrPluginUtils.setReturnFields(fl, rsp);
}
builder.setFieldFlags( fieldFlags );
rb.setFieldFlags( fieldFlags );
String defType = params.get(QueryParsing.DEFTYPE);
defType = defType==null ? OldLuceneQParserPlugin.NAME : defType;
if (builder.getQueryString() == null) {
builder.setQueryString( params.get( CommonParams.Q ) );
if (rb.getQueryString() == null) {
rb.setQueryString( params.get( CommonParams.Q ) );
}
QParser parser = QParser.getParser(builder.getQueryString(), defType, req);
builder.setQuery( parser.getQuery() );
builder.setSortSpec( parser.getSort(true) );
String[] fqs = req.getParams().getParams(org.apache.solr.common.params.CommonParams.FQ);
if (fqs!=null && fqs.length!=0) {
List<Query> filters = builder.getFilters();
if (filters==null) {
filters = new ArrayList<Query>();
builder.setFilters( filters );
}
for (String fq : fqs) {
if (fq != null && fq.trim().length()!=0) {
QParser fqp = QParser.getParser(fq, null, req);
filters.add(fqp.getQuery());
try {
QParser parser = QParser.getParser(rb.getQueryString(), defType, req);
rb.setQuery( parser.getQuery() );
rb.setSortSpec( parser.getSort(true) );
String[] fqs = req.getParams().getParams(org.apache.solr.common.params.CommonParams.FQ);
if (fqs!=null && fqs.length!=0) {
List<Query> filters = rb.getFilters();
if (filters==null) {
filters = new ArrayList<Query>();
rb.setFilters( filters );
}
for (String fq : fqs) {
if (fq != null && fq.trim().length()!=0) {
QParser fqp = QParser.getParser(fq, null, req);
filters.add(fqp.getQuery());
}
}
}
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
// TODO: temporary... this should go in a different component.
String shards = params.get("shards");
if (shards != null) {
List<String> lst = StrUtils.splitSmart(shards, ",", true);
rb.shards = lst.toArray(new String[lst.size()]);
}
}
@ -86,35 +111,692 @@ public class QueryComponent extends SearchComponent
* Actually run the query
*/
@Override
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
public void process(ResponseBuilder rb) throws IOException
{
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrIndexSearcher searcher = req.getSearcher();
SolrParams params = req.getParams();
if( builder.isNeedDocSet() ) {
builder.setResults( searcher.getDocListAndSet(
builder.getQuery(), builder.getFilters(), builder.getSortSpec().getSort(),
builder.getSortSpec().getOffset(), builder.getSortSpec().getCount(),
builder.getFieldFlags() ) );
// Optional: This could also be implemented by the top-level searcher sending
// a filter that lists the ids... that would be transparent to
// the request handler, but would be more expensive (and would preserve score
// too if desired).
String ids = params.get("ids");
if (ids != null) {
SchemaField idField = req.getSchema().getUniqueKeyField();
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
int[] luceneIds = new int[idArr.size()];
int docs = 0;
for (int i=0; i<idArr.size(); i++) {
int id = req.getSearcher().getFirstMatch(
new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
if (id >= 0)
luceneIds[docs++] = id;
}
DocListAndSet res = new DocListAndSet();
res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
if (rb.isNeedDocSet()) {
List<Query> queries = new ArrayList<Query>();
queries.add(rb.getQuery());
List<Query> filters = rb.getFilters();
if (filters != null) queries.addAll(filters);
res.docSet = searcher.getDocSet(queries);
}
rb.setResults(res);
rsp.add("response",rb.getResults().docList);
return;
}
if( rb.isNeedDocSet() ) {
rb.setResults( searcher.getDocListAndSet(
rb.getQuery(), rb.getFilters(), rb.getSortSpec().getSort(),
rb.getSortSpec().getOffset(), rb.getSortSpec().getCount(),
rb.getFieldFlags() ) );
}
else {
DocListAndSet results = new DocListAndSet();
results.docList = searcher.getDocList(
builder.getQuery(), builder.getFilters(), builder.getSortSpec().getSort(),
builder.getSortSpec().getOffset(), builder.getSortSpec().getCount(),
builder.getFieldFlags() );
builder.setResults( results );
rb.getQuery(), rb.getFilters(), rb.getSortSpec().getSort(),
rb.getSortSpec().getOffset(), rb.getSortSpec().getCount(),
rb.getFieldFlags() );
rb.setResults( results );
}
rsp.add("response",rb.getResults().docList);
boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES,false);
if(fsv){
Sort sort = rb.getSortSpec().getSort();
SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort();
ScoreDoc sd = new ScoreDoc(0,1.0f); // won't work for comparators that look at the score
NamedList sortVals = new NamedList();
StringFieldable field = new StringFieldable();
for (SortField sortField: sortFields) {
int type = sortField.getType();
if (type==SortField.SCORE || type==SortField.DOC) continue;
ScoreDocComparator comparator = null;
IndexReader reader = searcher.getReader();
String fieldname = sortField.getField();
FieldType ft = fieldname==null ? null : req.getSchema().getFieldTypeNoEx(fieldname);
switch (type) {
case SortField.INT:
comparator = comparatorInt (reader, fieldname);
break;
case SortField.FLOAT:
comparator = comparatorFloat (reader, fieldname);
break;
case SortField.LONG:
comparator = comparatorLong(reader, fieldname);
break;
case SortField.DOUBLE:
comparator = comparatorDouble(reader, fieldname);
break;
case SortField.STRING:
if (sortField.getLocale() != null) comparator = comparatorStringLocale (reader, fieldname, sortField.getLocale());
else comparator = comparatorString (reader, fieldname);
break;
case SortField.CUSTOM:
comparator = sortField.getFactory().newComparator (reader, fieldname);
break;
default:
throw new RuntimeException ("unknown field type: "+type);
}
DocList docList = rb.getResults().docList;
ArrayList<Object> vals = new ArrayList<Object>(docList.size());
DocIterator it = rb.getResults().docList.iterator();
while(it.hasNext()) {
sd.doc = it.nextDoc();
Object val = comparator.sortValue(sd);
// Sortable float, double, int, long types all just use a string
// comparator. For these, we need to put the type into a readable
// format. One reason for this is that XML can't represent all
// string values (or even all unicode code points).
// indexedToReadable() should be a no-op and should
// thus be harmless anyway (for all current ways anyway)
if (val instanceof String) {
field.val = (String)val;
val = ft.toObject(field);
}
vals.add(val);
}
sortVals.add(fieldname, vals);
}
rsp.add("sort_values", sortVals);
}
//pre-fetch returned documents
if (builder.getResults().docList != null && builder.getResults().docList.size()<=50) {
if (!req.getParams().getBool("isShard",false) && rb.getResults().docList != null && rb.getResults().docList.size()<=50) {
// TODO: this may depend on the highlighter component (or other components?)
SolrPluginUtils.optimizePreFetchDocs(builder.getResults().docList, builder.getQuery(), req, rsp);
SolrPluginUtils.optimizePreFetchDocs(rb.getResults().docList, rb.getQuery(), req, rsp);
}
rsp.add("response",builder.getResults().docList);
}
@Override
public int distributedProcess(ResponseBuilder rb) throws IOException {
if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY)
return ResponseBuilder.STAGE_PARSE_QUERY;
if (rb.stage == ResponseBuilder.STAGE_PARSE_QUERY) {
createDistributedIdf(rb);
return ResponseBuilder.STAGE_EXECUTE_QUERY;
}
if (rb.stage < ResponseBuilder.STAGE_EXECUTE_QUERY) return ResponseBuilder.STAGE_EXECUTE_QUERY;
if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) {
createMainQuery(rb);
return ResponseBuilder.STAGE_GET_FIELDS;
}
if (rb.stage < ResponseBuilder.STAGE_GET_FIELDS) return ResponseBuilder.STAGE_GET_FIELDS;
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
createRetrieveDocs(rb);
return ResponseBuilder.STAGE_DONE;
}
return ResponseBuilder.STAGE_DONE;
}
@Override
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
mergeIds(rb, sreq);
return;
}
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
returnFields(rb, sreq);
return;
}
}
@Override
public void finishStage(ResponseBuilder rb) {
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
// We may not have been able to retrieve all the docs due to an
// index change. Remove any null documents.
for (Iterator<SolrDocument> iter = rb._responseDocs.iterator(); iter.hasNext();) {
if (iter.next() == null) {
iter.remove();
rb._responseDocs.setNumFound(rb._responseDocs.getNumFound()-1);
}
}
rb.rsp.add("response", rb._responseDocs);
}
}
private void createDistributedIdf(ResponseBuilder rb) {
// TODO
}
private void createMainQuery(ResponseBuilder rb) {
ShardRequest sreq = new ShardRequest();
sreq.purpose = ShardRequest.PURPOSE_GET_TOP_IDS;
sreq.params = new ModifiableSolrParams(rb.req.getParams());
// TODO: base on current params or original params?
// don't pass through any shards param
sreq.params.remove("shards");
// set the start (offset) to 0 for each shard request so we can properly merge
// results from the start.
sreq.params.set("start","0");
// TODO: should we even use the SortSpec? That's obtained from the QParser, and
// perhaps we shouldn't attempt to parse the query at this level?
// Alternate Idea: instead of specifying all these things at the upper level,
// we could just specify that this is a shard request.
sreq.params.set("rows", rb.getSortSpec().getOffset() + rb.getSortSpec().getCount());
// in this first phase, request only the unique key field
// and any fields needed for merging.
sreq.params.set(ResponseBuilder.FIELD_SORT_VALUES,"true");
if (rb.getSortSpec().includesScore()) {
sreq.params.set("fl", rb.req.getSchema().getUniqueKeyField().getName() + ",score");
} else {
sreq.params.set("fl", rb.req.getSchema().getUniqueKeyField().getName());
}
rb.addRequest(this, sreq);
}
private void mergeIds(ResponseBuilder rb, ShardRequest sreq) {
SortSpec ss = rb.getSortSpec();
Sort sort = ss.getSort();
SortField[] sortFields = null;
if(sort != null) sortFields = sort.getSort();
else {
sortFields = new SortField[]{SortField.FIELD_SCORE};
}
SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField();
// id to shard mapping, to eliminate any accidental dups
HashMap<Object,String> uniqueDoc = new HashMap<Object,String>();
// Merge the docs via a priority queue so we don't have to sort *all* of the
// documents... we only need to order the top (rows+start)
ShardFieldSortedHitQueue queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount());
long numFound = 0;
Float maxScore=null;
for (ShardResponse srsp : sreq.responses) {
SolrDocumentList docs = (SolrDocumentList)srsp.rsp.getResponse().get("response");
// calculate global maxScore and numDocsFound
if (docs.getMaxScore() != null) {
maxScore = maxScore==null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore());
}
numFound += docs.getNumFound();
NamedList sortFieldValues = (NamedList)(srsp.rsp.getResponse().get("sort_values"));
// go through every doc in this response, construct a ShardDoc, and
// put it in the priority queue so it can be ordered.
for (int i=0; i<docs.size(); i++) {
SolrDocument doc = docs.get(i);
Object id = doc.getFieldValue(uniqueKeyField.getName());
String prevShard = uniqueDoc.put(id, srsp.shard);
if (prevShard != null) {
// duplicate detected
numFound--;
// For now, just always use the first encountered since we can't currently
// remove the previous one added to the priority queue. If we switched
// to the Java5 PriorityQueue, this would be easier.
continue;
// make which duplicate is used deterministic based on shard
// if (prevShard.compareTo(srsp.shard) >= 0) {
// TODO: remove previous from priority queue
// continue;
// }
}
ShardDoc shardDoc = new ShardDoc();
shardDoc.id = id;
shardDoc.shard = srsp.shard;
shardDoc.orderInShard = i;
Object scoreObj = doc.getFieldValue("score");
if (scoreObj != null) {
if (scoreObj instanceof String) {
shardDoc.score = Float.parseFloat((String)scoreObj);
} else {
shardDoc.score = (Float)scoreObj;
}
}
shardDoc.sortFieldValues = sortFieldValues;
queue.insert(shardDoc);
} // end for-each-doc-in-response
} // end for-each-response
// The queue now has 0 -> queuesize docs, where queuesize <= start + rows
// So we want to pop the last documents off the queue to get
// the docs offset -> queuesize
int resultSize = queue.size() - ss.getOffset();
resultSize = Math.max(0, resultSize); // there may not be any docs in range
Map<Object,ShardDoc> resultIds = new HashMap<Object,ShardDoc>();
for (int i=resultSize-1; i>=0; i--) {
ShardDoc shardDoc = (ShardDoc)queue.pop();
shardDoc.positionInResponse = i;
// Need the toString() for correlation with other lists that must
// be strings (like keys in highlighting, explain, etc)
resultIds.put(shardDoc.id.toString(), shardDoc);
}
SolrDocumentList responseDocs = new SolrDocumentList();
if (maxScore!=null) responseDocs.setMaxScore(maxScore);
responseDocs.setNumFound(numFound);
responseDocs.setStart(ss.getOffset());
// size appropriately
for (int i=0; i<resultSize; i++) responseDocs.add(null);
// save these results in a private area so we can access them
// again when retrieving stored fields.
// TODO: use ResponseBuilder (w/ comments) or the request context?
rb.resultIds = resultIds;
rb._responseDocs = responseDocs;
}
private void createRetrieveDocs(ResponseBuilder rb) {
// TODO: in a system with nTiers > 2, we could be passed "ids" here
// unless those requests always go to the final destination shard
// for each shard, collect the documents for that shard.
HashMap<String, Collection<ShardDoc>> shardMap = new HashMap<String,Collection<ShardDoc>>();
for (ShardDoc sdoc : rb.resultIds.values()) {
Collection<ShardDoc> shardDocs = shardMap.get(sdoc.shard);
if (shardDocs == null) {
shardDocs = new ArrayList<ShardDoc>();
shardMap.put(sdoc.shard, shardDocs);
}
shardDocs.add(sdoc);
}
SchemaField uniqueField = rb.req.getSchema().getUniqueKeyField();
// Now create a request for each shard to retrieve the stored fields
for (Collection<ShardDoc> shardDocs : shardMap.values()) {
ShardRequest sreq = new ShardRequest();
sreq.purpose = ShardRequest.PURPOSE_GET_FIELDS;
sreq.shards = new String[] {shardDocs.iterator().next().shard};
sreq.params = new ModifiableSolrParams();
// add original params
sreq.params.add( rb.req.getParams());
// no need for a sort, we already have order
sreq.params.remove("sort");
// we already have the field sort values
sreq.params.remove(ResponseBuilder.FIELD_SORT_VALUES);
// make sure that the id is returned for correlation
String fl = sreq.params.get("fl");
if (fl != null) {
sreq.params.set("fl", fl+','+uniqueField.getName());
}
ArrayList<String> ids = new ArrayList<String>(shardDocs.size());
for (ShardDoc shardDoc : shardDocs) {
// TODO: depending on the type, we may need more tha a simple toString()?
ids.add(shardDoc.id.toString());
}
sreq.params.add("ids", StrUtils.join(ids, ','));
rb.addRequest(this, sreq);
}
}
private void returnFields(ResponseBuilder rb, ShardRequest sreq) {
// Keep in mind that this could also be a shard in a multi-tiered system.
// TODO: if a multi-tiered system, it seems like some requests
// could/should bypass middlemen (like retrieving stored fields)
// TODO: merge fsv to if requested
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
boolean returnScores = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0;
assert(sreq.responses.size() == 1);
ShardResponse srsp = sreq.responses.get(0);
SolrDocumentList docs = (SolrDocumentList)srsp.rsp.getResponse().get("response");
String keyFieldName = rb.req.getSchema().getUniqueKeyField().getName();
for (SolrDocument doc : docs) {
Object id = doc.getFieldValue(keyFieldName);
ShardDoc sdoc = rb.resultIds.get(id.toString());
if (returnScores && sdoc.score != null) {
doc.setField("score", sdoc.score);
}
rb._responseDocs.set(sdoc.positionInResponse, doc);
}
}
}
/////////////////////////////////////////////
/// Comparators copied from Lucene
/////////////////////////////////////////////
/**
* Returns a comparator for sorting hits according to a field containing integers.
* @param reader Index to use.
* @param fieldname Fieldable containg integer values.
* @return Comparator for sorting hits.
* @throws IOException If an error occurs reading the index.
*/
static ScoreDocComparator comparatorInt (final IndexReader reader, final String fieldname)
throws IOException {
final String field = fieldname.intern();
final int[] fieldOrder = FieldCache.DEFAULT.getInts (reader, field);
return new ScoreDocComparator() {
public final int compare (final ScoreDoc i, final ScoreDoc j) {
final int fi = fieldOrder[i.doc];
final int fj = fieldOrder[j.doc];
if (fi < fj) return -1;
if (fi > fj) return 1;
return 0;
}
public Comparable sortValue (final ScoreDoc i) {
return new Integer (fieldOrder[i.doc]);
}
public int sortType() {
return SortField.INT;
}
};
}
/**
* Returns a comparator for sorting hits according to a field containing integers.
* @param reader Index to use.
* @param fieldname Fieldable containg integer values.
* @return Comparator for sorting hits.
* @throws IOException If an error occurs reading the index.
*/
static ScoreDocComparator comparatorLong (final IndexReader reader, final String fieldname)
throws IOException {
final String field = fieldname.intern();
final long[] fieldOrder = ExtendedFieldCache.EXT_DEFAULT.getLongs (reader, field);
return new ScoreDocComparator() {
public final int compare (final ScoreDoc i, final ScoreDoc j) {
final long li = fieldOrder[i.doc];
final long lj = fieldOrder[j.doc];
if (li < lj) return -1;
if (li > lj) return 1;
return 0;
}
public Comparable sortValue (final ScoreDoc i) {
return new Long(fieldOrder[i.doc]);
}
public int sortType() {
return SortField.LONG;
}
};
}
/**
* Returns a comparator for sorting hits according to a field containing floats.
* @param reader Index to use.
* @param fieldname Fieldable containg float values.
* @return Comparator for sorting hits.
* @throws IOException If an error occurs reading the index.
*/
static ScoreDocComparator comparatorFloat (final IndexReader reader, final String fieldname)
throws IOException {
final String field = fieldname.intern();
final float[] fieldOrder = FieldCache.DEFAULT.getFloats (reader, field);
return new ScoreDocComparator () {
public final int compare (final ScoreDoc i, final ScoreDoc j) {
final float fi = fieldOrder[i.doc];
final float fj = fieldOrder[j.doc];
if (fi < fj) return -1;
if (fi > fj) return 1;
return 0;
}
public Comparable sortValue (final ScoreDoc i) {
return new Float (fieldOrder[i.doc]);
}
public int sortType() {
return SortField.FLOAT;
}
};
}
/**
* Returns a comparator for sorting hits according to a field containing doubles.
* @param reader Index to use.
* @param fieldname Fieldable containg float values.
* @return Comparator for sorting hits.
* @throws IOException If an error occurs reading the index.
*/
static ScoreDocComparator comparatorDouble(final IndexReader reader, final String fieldname)
throws IOException {
final String field = fieldname.intern();
final double[] fieldOrder = ExtendedFieldCache.EXT_DEFAULT.getDoubles (reader, field);
return new ScoreDocComparator () {
public final int compare (final ScoreDoc i, final ScoreDoc j) {
final double di = fieldOrder[i.doc];
final double dj = fieldOrder[j.doc];
if (di < dj) return -1;
if (di > dj) return 1;
return 0;
}
public Comparable sortValue (final ScoreDoc i) {
return new Double (fieldOrder[i.doc]);
}
public int sortType() {
return SortField.DOUBLE;
}
};
}
/**
* Returns a comparator for sorting hits according to a field containing strings.
* @param reader Index to use.
* @param fieldname Fieldable containg string values.
* @return Comparator for sorting hits.
* @throws IOException If an error occurs reading the index.
*/
static ScoreDocComparator comparatorString (final IndexReader reader, final String fieldname)
throws IOException {
final String field = fieldname.intern();
final FieldCache.StringIndex index = FieldCache.DEFAULT.getStringIndex (reader, field);
return new ScoreDocComparator () {
public final int compare (final ScoreDoc i, final ScoreDoc j) {
final int fi = index.order[i.doc];
final int fj = index.order[j.doc];
if (fi < fj) return -1;
if (fi > fj) return 1;
return 0;
}
public Comparable sortValue (final ScoreDoc i) {
return index.lookup[index.order[i.doc]];
}
public int sortType() {
return SortField.STRING;
}
};
}
/**
* Returns a comparator for sorting hits according to a field containing strings.
* @param reader Index to use.
* @param fieldname Fieldable containg string values.
* @return Comparator for sorting hits.
* @throws IOException If an error occurs reading the index.
*/
static ScoreDocComparator comparatorStringLocale (final IndexReader reader, final String fieldname, final Locale locale)
throws IOException {
final Collator collator = Collator.getInstance (locale);
final String field = fieldname.intern();
final String[] index = FieldCache.DEFAULT.getStrings (reader, field);
return new ScoreDocComparator() {
public final int compare(final ScoreDoc i, final ScoreDoc j) {
String is = index[i.doc];
String js = index[j.doc];
if (is == js) {
return 0;
} else if (is == null) {
return -1;
} else if (js == null) {
return 1;
} else {
return collator.compare(is, js);
}
}
public Comparable sortValue (final ScoreDoc i) {
return index[i.doc];
}
public int sortType() {
return SortField.STRING;
}
};
}
static class StringFieldable implements Fieldable {
public String val;
public void setBoost(float boost) {
}
public float getBoost() {
return 0;
}
public String name() {
return null;
}
public String stringValue() {
return val;
}
public Reader readerValue() {
return null;
}
public byte[] binaryValue() {
return new byte[0];
}
public TokenStream tokenStreamValue() {
return null;
}
public boolean isStored() {
return true;
}
public boolean isIndexed() {
return true;
}
public boolean isTokenized() {
return true;
}
public boolean isCompressed() {
return false;
}
public boolean isTermVectorStored() {
return false;
}
public boolean isStoreOffsetWithTermVector() {
return false;
}
public boolean isStorePositionWithTermVector() {
return false;
}
public boolean isBinary() {
return false;
}
public boolean getOmitNorms() {
return false;
}
public void setOmitNorms(boolean omitNorms) {
}
public boolean isLazy() {
return false;
}
}
/////////////////////////////////////////////
/// SolrInfoMBean

22
src/java/org/apache/solr/handler/component/QueryElevationComponent.java
View File

@ -25,7 +25,6 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -59,13 +58,12 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.Config;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SortSpec;
import org.apache.solr.util.VersionedFile;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.apache.solr.request.SolrQueryRequest;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@ -304,22 +302,22 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
//---------------------------------------------------------------------------------
@Override
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
public void prepare(ResponseBuilder rb) throws IOException
{
SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
// A runtime param can skip
if( !params.getBool( ENABLE, true ) ) {
return;
}
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
Query query = builder.getQuery();
Query query = rb.getQuery();
if( query == null ) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"The QueryElevationComponent needs to be registered 'after' the query component" );
}
String qstr = getAnalyzedQuery( builder.getQueryString() );
String qstr = getAnalyzedQuery( rb.getQueryString() );
IndexReader reader = req.getSearcher().getReader();
ElevationObj booster = null;
try {
@ -340,11 +338,11 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
newq.add( bq );
}
}
builder.setQuery( newq );
rb.setQuery( newq );
// if the sort is 'score desc' use a custom sorting method to
// insert documents in their proper place
SortSpec sortSpec = builder.getSortSpec();
SortSpec sortSpec = rb.getSortSpec();
if( sortSpec.getSort() == null ) {
sortSpec.setSort( new Sort( new SortField[] {
new SortField(idField, new ElevationComparatorSource(booster.priority), false ),
@ -377,7 +375,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
}
// Add debugging information
if( builder.isDebug() ) {
if( rb.isDebug() ) {
List<String> match = null;
if( booster != null ) {
// Extract the elevated terms into a list
@ -391,12 +389,12 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
SimpleOrderedMap<Object> dbg = new SimpleOrderedMap<Object>();
dbg.add( "q", qstr );
dbg.add( "match", match );
builder.addDebugInfo( "queryBoosting", dbg );
rb.addDebugInfo( "queryBoosting", dbg );
}
}
@Override
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
public void process(ResponseBuilder rb) throws IOException {
// Do nothing -- the real work is modifying the input query
}

99
src/java/org/apache/solr/handler/component/ResponseBuilder.java
View File

@ -18,23 +18,34 @@
package org.apache.solr.handler.component;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.RTimer;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.search.DocListAndSet;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SortSpec;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* TODO!
* This class is experimental and will be changing in the future.
*
* @version $Id$
* @since solr 1.3
*/
public class ResponseBuilder
{
public SolrQueryRequest req;
public SolrQueryResponse rsp;
public boolean doHighlights;
public boolean doFacets;
private boolean needDocList = false;
private boolean needDocSet = false;
private int fieldFlags = 0;
@ -51,11 +62,74 @@ public class ResponseBuilder
private RTimer timer = null;
private Query highlightQuery = null;
//-------------------------------------------------------------------------
//-------------------------------------------------------------------------
public List<SearchComponent> components;
//////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////
//// Distributed Search section
//////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////
public static final String FIELD_SORT_VALUES = "fsv";
public static final String SHARDS = "shards";
public static final String IDS = "ids";
/***
public static final String NUMDOCS = "nd";
public static final String DOCFREQS = "tdf";
public static final String TERMS = "terms";
public static final String EXTRACT_QUERY_TERMS = "eqt";
public static final String LOCAL_SHARD = "local";
public static final String DOC_QUERY = "dq";
***/
public static int STAGE_START = 0;
public static int STAGE_PARSE_QUERY = 1000;
public static int STAGE_EXECUTE_QUERY = 2000;
public static int STAGE_GET_FIELDS = 3000;
public static int STAGE_DONE = Integer.MAX_VALUE;
public int stage; // What stage is this current request at?
public String[] shards;
public List<ShardRequest> outgoing; // requests to be sent
public List<ShardRequest> finished; // requests that have received responses from all shards
public int getShardNum(String shard) {
for (int i=0; i<shards.length; i++) {
if (shards[i]==shard || shards[i].equals(shard)) return i;
}
return -1;
}
public void addRequest(SearchComponent me, ShardRequest sreq) {
outgoing.add(sreq);
if ((sreq.purpose & ShardRequest.PURPOSE_PRIVATE)==0) {
// if this isn't a private request, let other components modify it.
for (SearchComponent component : components) {
if (component != me) {
component.modifyRequest(this, me, sreq);
}
}
}
}
public GlobalCollectionStat globalCollectionStat;
Map<Object, ShardDoc> resultIds;
// Maps uniqueKeyValue to ShardDoc, which may be used to
// determine order of the doc or uniqueKey in the final
// returned sequence.
// Only valid after STAGE_EXECUTE_QUERY has completed.
/* private... components that don't own these shouldn't use them */
SolrDocumentList _responseDocs;
FacetInfo _facetInfo;
/**
* Utility function to add debugging info. This will make sure a valid
* debugInfo exists before adding to it.
@ -70,7 +144,7 @@ public class ResponseBuilder
//-------------------------------------------------------------------------
//-------------------------------------------------------------------------
public boolean isDebug() {
return debug;
}
@ -174,4 +248,17 @@ public class ResponseBuilder
public void setTimer(RTimer timer) {
this.timer = timer;
}
public static class GlobalCollectionStat {
public final long numDocs;
public final Map<String, Long> dfMap;
public GlobalCollectionStat(int numDocs, Map<String, Long> dfMap) {
this.numDocs = numDocs;
this.dfMap = dfMap;
}
}
}

26
src/java/org/apache/solr/handler/component/SearchComponent.java
View File

@ -36,11 +36,31 @@ import org.apache.solr.util.plugin.NamedListInitializedPlugin;
*/
public abstract class SearchComponent implements SolrInfoMBean, NamedListInitializedPlugin
{
public abstract void prepare( SolrQueryRequest req, SolrQueryResponse rsp ) throws IOException, ParseException;
public abstract void process( SolrQueryRequest req, SolrQueryResponse rsp ) throws IOException;
public abstract void prepare(ResponseBuilder rb) throws IOException;
public abstract void process(ResponseBuilder rb) throws IOException;
/** Process for a distributed search.
* @returns the next stage for this component */
public int distributedProcess(ResponseBuilder rb) throws IOException {
return ResponseBuilder.STAGE_DONE;
}
/** Called after another component adds a request */
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
}
/** Called after all responses for a single request were received */
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
}
/** Called after all responses have been received for this stage.
* Useful when different requests are sent to each shard.
*/
public void finishStage(ResponseBuilder rb) {
}
//////////////////////// NamedListInitializedPlugin methods //////////////////////
public void init( NamedList args )
{
// By default do nothing

358
src/java/org/apache/solr/handler/component/SearchHandler.java
View File

@ -17,41 +17,45 @@
package org.apache.solr.handler.component;
import org.apache.lucene.queryParser.ParseException;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.RTimer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.SolrException;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.apache.solr.core.SolrCore;
import org.apache.lucene.queryParser.ParseException;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.HttpClient;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;
import java.util.*;
import java.util.concurrent.*;
/**
*
* Refer SOLR-281
*
* @since solr 1.3
*/
public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
{
static final String RESPONSE_BUILDER_CONTEXT_KEY = "ResponseBuilder";
static final String INIT_COMPONENTS = "components";
static final String INIT_FIRST_COMPONENTS = "first-components";
static final String INIT_LAST_COMPONENTS = "last-components";
protected static Logger log = Logger.getLogger(SearchHandler.class.getName());
protected List<SearchComponent> components = null;
protected NamedList initArgs = null;
@Override
public void init(NamedList args) {
super.init( args );
@ -73,7 +77,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
* Initialize the components based on name
*/
@SuppressWarnings("unchecked")
public void inform(SolrCore core)
public void inform(SolrCore core)
{
Object declaredComponents = initArgs.get(INIT_COMPONENTS);
List<String> first = (List<String>) initArgs.get(INIT_FIRST_COMPONENTS);
@ -83,13 +87,13 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
if( declaredComponents == null ) {
// Use the default component list
list = getDefaultComponets();
if( first != null ) {
List<String> clist = first;
clist.addAll( list );
list = clist;
}
if( last != null ) {
list.addAll( last );
}
@ -101,7 +105,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
"First/Last components only valid if you do not declare 'components'");
}
}
// Build the component list
components = new ArrayList<SearchComponent>( list.size() );
for(String c : list){
@ -115,71 +119,147 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
return components;
}
public static ResponseBuilder getResponseBuilder(SolrQueryRequest req)
{
return (ResponseBuilder) req.getContext().get( RESPONSE_BUILDER_CONTEXT_KEY );
}
//---------------------------------------------------------------------------------------
// SolrRequestHandler
//---------------------------------------------------------------------------------------
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, ParseException, InstantiationException, IllegalAccessException
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception, ParseException, InstantiationException, IllegalAccessException
{
ResponseBuilder builder = new ResponseBuilder();
req.getContext().put( RESPONSE_BUILDER_CONTEXT_KEY, builder );
if( components == null ) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"SearchHandler not initialized properly. No components registered." );
}
// The semantics of debugging vs not debugging are distinct enough
// to justify two control loops
if( !req.getParams().getBool( CommonParams.DEBUG_QUERY, false ) ) {
// Prepare
ResponseBuilder rb = new ResponseBuilder();
rb.req = req;
rb.rsp = rsp;
rb.components = components;
rb.setDebug(req.getParams().getBool(CommonParams.DEBUG_QUERY, false));
final RTimer timer = rb.isDebug() ? new RTimer() : null;
if (timer == null) {
// non-debugging prepare phase
for( SearchComponent c : components ) {
c.prepare( req, rsp );
c.prepare(rb);
}
// Process
for( SearchComponent c : components ) {
c.process( req, rsp );
}
}
else {
builder.setDebug( true );
RTimer timer = new RTimer();
// Prepare
} else {
// debugging prepare phase
RTimer subt = timer.sub( "prepare" );
for( SearchComponent c : components ) {
builder.setTimer( subt.sub( c.getName() ) );
c.prepare( req, rsp );
builder.getTimer().stop();
rb.setTimer( subt.sub( c.getName() ) );
c.prepare(rb);
rb.getTimer().stop();
}
subt.stop();
// Process
subt = timer.sub( "process" );
for( SearchComponent c : components ) {
builder.setTimer( subt.sub( c.getName() ) );
c.process( req, rsp );
builder.getTimer().stop();
}
if (rb.shards == null) {
// a normal non-distributed request
// The semantics of debugging vs not debugging are different enough that
// it makes sense to have two control loops
if(!rb.isDebug()) {
// Process
for( SearchComponent c : components ) {
c.process(rb);
}
}
subt.stop();
timer.stop();
// add the timing info
builder.addDebugInfo( "timing", timer.asNamedList() );
else {
// Process
RTimer subt = timer.sub( "process" );
for( SearchComponent c : components ) {
rb.setTimer( subt.sub( c.getName() ) );
c.process(rb);
rb.getTimer().stop();
}
subt.stop();
timer.stop();
// add the timing info
if( rb.getDebugInfo() == null ) {
rb.setDebugInfo( new SimpleOrderedMap<Object>() );
}
rb.getDebugInfo().add( "timing", timer.asNamedList() );
}
} else {
// a distributed request
HttpCommComponent comm = new HttpCommComponent();
if (rb.outgoing == null) {
rb.outgoing = new LinkedList<ShardRequest>();
}
rb.finished = new ArrayList<ShardRequest>();
int nextStage = 0;
do {
rb.stage = nextStage;
nextStage = ResponseBuilder.STAGE_DONE;
// call all components
for( SearchComponent c : components ) {
// the next stage is the minimum of what all components report
nextStage = Math.min(nextStage, c.distributedProcess(rb));
}
// check the outgoing queue and send requests
while (rb.outgoing.size() > 0) {
// submit all current request tasks at once
while (rb.outgoing.size() > 0) {
ShardRequest sreq = rb.outgoing.remove(0);
sreq.actualShards = sreq.shards;
if (sreq.actualShards==ShardRequest.ALL_SHARDS) {
sreq.actualShards = rb.shards;
}
sreq.responses = new ArrayList<ShardResponse>();
// TODO: map from shard to address[]
for (String shard : sreq.actualShards) {
ModifiableSolrParams params = sreq.params;
params.remove("shards"); // not a top-level request
params.remove("indent");
params.remove("echoParams");
params.set("isShard", true); // a sub (shard) request
comm.submit(sreq, shard, params);
}
}
// now wait for replies, but if anyone puts more requests on
// the outgoing queue, send them out immediately (by exiting
// this loop)
while (rb.outgoing.size() == 0) {
ShardResponse srsp = comm.takeCompletedOrError();
if (srsp == null) break; // no more requests to wait for
// Was there an exception? If so, abort everything and
// rethrow
if (srsp.exception != null) {
comm.cancelAll();
if (srsp.exception instanceof SolrException) {
throw (SolrException)srsp.exception;
} else {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, srsp.exception);
}
}
rb.finished.add(srsp.req);
// let the components see the responses to the request
for(SearchComponent c : components) {
c.handleResponses(rb, srsp.req);
}
}
}
for(SearchComponent c : components) {
c.finishStage(rb);
}
// we are done when the next stage is MAX_VALUE
} while (nextStage != Integer.MAX_VALUE);
}
}
//---------------------------------------------------------------------------------------
// SolrInfoMBeans
//---------------------------------------------------------------------------------------
//////////////////////// SolrInfoMBeans methods //////////////////////
@Override
public String getDescription() {
StringBuilder sb = new StringBuilder();
@ -206,3 +286,139 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
return "$URL$";
}
}
// TODO: generalize how a comm component can fit into search component framework
// TODO: statics should be per-core singletons
class HttpCommComponent {
// We want an executor that doesn't take up any resources if
// it's not used, so it could be created statically for
// the distributed search component if desired.
//
// Consider CallerRuns policy and a lower max threads to throttle
// requests at some point (or should we simply return failure?)
static Executor commExecutor = new ThreadPoolExecutor(
0,
Integer.MAX_VALUE,
5, TimeUnit.SECONDS, // terminate idle threads after 5 sec
new SynchronousQueue<Runnable>() // directly hand off tasks
);
static HttpClient client;
static {
MultiThreadedHttpConnectionManager mgr = new MultiThreadedHttpConnectionManager();
mgr.getParams().setDefaultMaxConnectionsPerHost(20);
mgr.getParams().setMaxTotalConnections(10000);
// mgr.getParams().setStaleCheckingEnabled(false);
client = new HttpClient(mgr);
}
CompletionService<ShardResponse> completionService = new ExecutorCompletionService<ShardResponse>(commExecutor);
Set<Future<ShardResponse>> pending = new HashSet<Future<ShardResponse>>();
HttpCommComponent() {
}
void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params) {
Callable<ShardResponse> task = new Callable<ShardResponse>() {
public ShardResponse call() throws Exception {
ShardResponse srsp = new ShardResponse();
srsp.req = sreq;
srsp.shard = shard;
try {
// String url = "http://" + shard + "/select";
String url = "http://" + shard;
params.remove("wt"); // use default (or should we explicitly set it?)
params.remove("version");
SolrServer server = new CommonsHttpSolrServer(url, client);
// SolrRequest req = new SolrRequest(SolrRequest.METHOD.GET, "/select");
// use generic request to avoid extra processing of queries
// QueryRequest req = new QueryRequest(sreq.params);
// srsp.rsp = server.request(req);
srsp.rsp = server.query(sreq.params);
} catch (Throwable th) {
srsp.exception = th;
if (th instanceof SolrException) {
srsp.rspCode = ((SolrException)th).code();
} else {
srsp.rspCode = -1;
}
}
return srsp;
}
};
pending.add( completionService.submit(task) );
}
/** returns a ShardResponse of the last response correlated with a ShardRequest */
ShardResponse take() {
while (pending.size() > 0) {
try {
Future<ShardResponse> future = completionService.take();
pending.remove(future);
ShardResponse rsp = future.get();
rsp.req.responses.add(rsp);
if (rsp.req.responses.size() == rsp.req.actualShards.length) {
return rsp;
}
} catch (InterruptedException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} catch (ExecutionException e) {
// should be impossible... the problem with catching the exception
// at this level is we don't know what ShardRequest it applied to
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Impossible Exception",e);
}
}
return null;
}
/** returns a ShardResponse of the last response correlated with a ShardRequest,
* or immediately returns a ShardResponse if there was an error detected
*/
ShardResponse takeCompletedOrError() {
while (pending.size() > 0) {
try {
Future<ShardResponse> future = completionService.take();
pending.remove(future);
ShardResponse rsp = future.get();
if (rsp.exception != null) return rsp; // if exception, return immediately
// add response to the response list... we do this after the take() and
// not after the completion of "call" so we know when the last response
// for a request was received. Otherwise we might return the same
// request more than once.
rsp.req.responses.add(rsp);
if (rsp.req.responses.size() == rsp.req.actualShards.length) {
return rsp;
}
} catch (InterruptedException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} catch (ExecutionException e) {
// should be impossible... the problem with catching the exception
// at this level is we don't know what ShardRequest it applied to
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Impossible Exception",e);
}
}
return null;
}
void cancelAll() {
for (Future<ShardResponse> future : pending) {
// TODO: any issues with interrupting? shouldn't be if
// there are finally blocks to release connections.
future.cancel(true);
}
}
}

269
src/java/org/apache/solr/handler/component/ShardDoc.java Executable file
View File

@ -0,0 +1,269 @@
package org.apache.solr.handler.component;
import org.apache.lucene.search.SortComparatorSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.PriorityQueue;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.search.MissingStringLastComparatorSource;
import java.text.Collator;
import java.util.Comparator;
import java.util.Locale;
import java.util.List;
import java.util.ArrayList;
public class ShardDoc {
public String shard;
public String shardAddress; // TODO
int orderInShard;
// the position of this doc within the shard... this can be used
// to short-circuit comparisons if the shard is equal, and can
// also be used to break ties within the same shard.
Object id;
// this is currently the uniqueKeyField but
// may be replaced with internal docid in a future release.
Float score;
NamedList sortFieldValues;
// sort field values for *all* docs in a particular shard.
// this doc's values are in position orderInShard
// TODO: store the SolrDocument here?
// Store the order in the merged list for lookup when getting stored fields?
// (other components need this ordering to store data in order, like highlighting)
// but we shouldn't expose uniqueKey (have a map by it) until the stored-field
// retrieval stage.
int positionInResponse;
// the ordinal position in the merged response arraylist
public String toString(){
return "id="+id
+" ,score="+score
+" ,shard="+shard
+" ,orderInShard="+orderInShard
+" ,positionInResponse="+positionInResponse
+" ,sortFieldValues="+sortFieldValues;
}
}
// used by distributed search to merge results.
class ShardFieldSortedHitQueue extends PriorityQueue {
/** Stores a comparator corresponding to each field being sorted by */
protected Comparator[] comparators;
/** Stores the sort criteria being used. */
protected SortField[] fields;
/** The order of these fieldNames should correspond to the order of sort field values retrieved from the shard */
protected List<String> fieldNames = new ArrayList<String>();
public ShardFieldSortedHitQueue(SortField[] fields, int size) {
final int n = fields.length;
comparators = new Comparator[n];
this.fields = new SortField[n];
for (int i = 0; i < n; ++i) {
// keep track of the named fields
int type = fields[i].getType();
if (type!=SortField.SCORE && type!=SortField.DOC) {
fieldNames.add(fields[i].getField());
}
String fieldname = fields[i].getField();
comparators[i] = getCachedComparator(fieldname, fields[i]
.getType(), fields[i].getLocale(), fields[i].getFactory());
if (fields[i].getType() == SortField.STRING) {
this.fields[i] = new SortField(fieldname, fields[i].getLocale(),
fields[i].getReverse());
} else {
this.fields[i] = new SortField(fieldname, fields[i].getType(),
fields[i].getReverse());
}
//System.out.println("%%%%%%%%%%%%%%%%%% got "+fields[i].getType() +" for "+ fieldname +" fields[i].getReverse(): "+fields[i].getReverse());
}
initialize(size);
}
@Override
protected boolean lessThan(Object objA, Object objB) {
ShardDoc docA = (ShardDoc)objA;
ShardDoc docB = (ShardDoc)objB;
// If these docs are from the same shard, then the relative order
// is how they appeared in the response from that shard.
if (docA.shard == docB.shard) {
// if docA has a smaller position, it should be "larger" so it
// comes before docB.
// This will handle sorting by docid within the same shard
// comment this out to test comparators.
return !(docA.orderInShard < docB.orderInShard);
}
// run comparators
final int n = comparators.length;
int c = 0;
for (int i = 0; i < n && c == 0; i++) {
c = (fields[i].getReverse()) ? comparators[i].compare(docB, docA)
: comparators[i].compare(docA, docB);
}
// solve tiebreaks by comparing shards (similar to using docid)
// smaller docid's beat larger ids, so reverse the natural ordering
if (c == 0) {
c = -docA.shard.compareTo(docB.shard);
}
return c < 0;
}
Comparator getCachedComparator(String fieldname, int type, Locale locale, SortComparatorSource factory) {
Comparator comparator = null;
switch (type) {
case SortField.SCORE:
comparator = comparatorScore(fieldname);
break;
case SortField.STRING:
if (locale != null)
comparator = comparatorStringLocale(fieldname, locale);
else
comparator = comparatorNatural(fieldname);
break;
case SortField.CUSTOM:
if (factory instanceof MissingStringLastComparatorSource){
comparator = comparatorMissingStringLast(fieldname);
} else {
// TODO: support other types such as random... is there a way to
// support generically? Perhaps just comparing Object
comparator = comparatorNatural(fieldname);
// throw new RuntimeException("Custom sort not supported factory is "+factory.getClass());
}
break;
case SortField.DOC:
// TODO: we can support this!
throw new RuntimeException("Doc sort not supported");
default:
comparator = comparatorNatural(fieldname);
break;
}
return comparator;
}
class ShardComparator implements Comparator {
String fieldName;
int fieldNum;
public ShardComparator(String fieldName) {
this.fieldName = fieldName;
this.fieldNum=0;
for (int i=0; i<fieldNames.size(); i++) {
if (fieldNames.get(i).equals(fieldName)) {
this.fieldNum = i;
break;
}
}
}
Object sortVal(ShardDoc shardDoc) {
assert(shardDoc.sortFieldValues.getName(fieldNum).equals(fieldName));
List lst = (List)shardDoc.sortFieldValues.getVal(fieldNum);
return lst.get(shardDoc.orderInShard);
}
public int compare(Object o1, Object o2) {
return 0;
}
}
static Comparator comparatorScore(final String fieldName) {
return new Comparator() {
public final int compare(final Object o1, final Object o2) {
ShardDoc e1 = (ShardDoc) o1;
ShardDoc e2 = (ShardDoc) o2;
final float f1 = e1.score;
final float f2 = e2.score;
if (f1 < f2)
return -1;
if (f1 > f2)
return 1;
return 0;
}
};
}
// The lucene natural sort ordering corresponds to numeric
// and string natural sort orderings (ascending). Since
// the PriorityQueue keeps the biggest elements by default,
// we need to reverse the natural compare ordering so that the
// smallest elements are kept instead of the largest... hence
// the negative sign on the final compareTo().
Comparator comparatorNatural(String fieldName) {
return new ShardComparator(fieldName) {
public final int compare(final Object o1, final Object o2) {
ShardDoc sd1 = (ShardDoc) o1;
ShardDoc sd2 = (ShardDoc) o2;
Comparable v1 = (Comparable)sortVal(sd1);
Comparable v2 = (Comparable)sortVal(sd2);
if (v1==v2)
return 0;
if (v1==null)
return 1;
if(v2==null)
return -1;
return -v1.compareTo(v2);
}
};
}
Comparator comparatorStringLocale(final String fieldName,
Locale locale) {
final Collator collator = Collator.getInstance(locale);
return new ShardComparator(fieldName) {
public final int compare(final Object o1, final Object o2) {
ShardDoc sd1 = (ShardDoc) o1;
ShardDoc sd2 = (ShardDoc) o2;
Comparable v1 = (Comparable)sortVal(sd1);
Comparable v2 = (Comparable)sortVal(sd2);
if (v1==v2)
return 0;
if (v1==null)
return 1;
if(v2==null)
return -1;
return -collator.compare(v1,v2);
}
};
}
Comparator comparatorMissingStringLast(final String fieldName) {
return new ShardComparator(fieldName) {
public final int compare(final Object o1, final Object o2) {
ShardDoc sd1 = (ShardDoc) o1;
ShardDoc sd2 = (ShardDoc) o2;
Comparable v1 = (Comparable)sortVal(sd1);
Comparable v2 = (Comparable)sortVal(sd2);
if (v1==v2)
return 0;
if (v1==null)
return -1;
if(v2==null)
return 1;
return -v1.compareTo(v2);
}
};
}
}

70
src/java/org/apache/solr/handler/component/ShardRequest.java Executable file
View File

@ -0,0 +1,70 @@
package org.apache.solr.handler.component;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import java.util.ArrayList;
import java.util.List;
// todo... when finalized make accessors
public class ShardRequest {
public final static String[] ALL_SHARDS = null;
public final static int PURPOSE_PRIVATE = 0x01;
public final static int PURPOSE_GET_TERM_DFS = 0x02;
public final static int PURPOSE_GET_TOP_IDS = 0x04;
public final static int PURPOSE_REFINE_TOP_IDS = 0x08;
public final static int PURPOSE_GET_FACETS = 0x10;
public final static int PURPOSE_REFINE_FACETS = 0x20;
public final static int PURPOSE_GET_FIELDS = 0x40;
public final static int PURPOSE_GET_HIGHLIGHTS = 0x80;
public final static int PURPOSE_GET_DEBUG =0x100;
public int purpose; // the purpose of this request
public String[] shards; // the shards this request should be sent to, null for all
// TODO: how to request a specific shard address?
public ModifiableSolrParams params;
/** list of responses... filled out by framework */
public List<ShardResponse> responses = new ArrayList<ShardResponse>();
/** actual shards to send the request to, filled out by framework */
public String[] actualShards;
// TODO: one could store a list of numbers to correlate where returned docs
// go in the top-level response rather than looking up by id...
// this would work well if we ever transitioned to using internal ids and
// didn't require a uniqueId
public String toString() {
return "ShardRequest:{params=" + params
+ ", purpose=" + Integer.toHexString(purpose)
+ ", nResponses =" + responses.size()
+ "}";
}
}
class ShardResponse {
public ShardRequest req;
public String shard;
public String shardAddress; // the specific shard that this response was received from
public int rspCode;
public Throwable exception;
public SolrResponse rsp;
public String toString() {
return "ShardResponse:{shard="+shard+",shardAddress="+shardAddress
+"\n\trequest=" + req
+"\n\tresponse=" + rsp
+ (exception==null ? "" : "\n\texception="+ SolrException.toStr(exception))
+"\n}";
}
}

12
src/java/org/apache/solr/request/SimpleFacets.java
View File

@ -62,12 +62,13 @@ public class SimpleFacets {
protected SolrParams params;
/** Searcher to use for all calculations */
protected SolrIndexSearcher searcher;
protected SolrQueryRequest req;
public SimpleFacets(SolrIndexSearcher searcher,
public SimpleFacets(SolrQueryRequest req,
DocSet docs,
SolrParams params) {
this.searcher = searcher;
this.req = req;
this.searcher = req.getSearcher();
this.docs = docs;
this.params = params;
}
@ -117,12 +118,13 @@ public class SimpleFacets {
* If user doesn't want schema default for facet.query, they should be
* explicit.
*/
SolrQueryParser qp = searcher.getSchema().getSolrQueryParser(null);
// SolrQueryParser qp = searcher.getSchema().getSolrQueryParser(null);
String[] facetQs = params.getParams(FacetParams.FACET_QUERY);
if (null != facetQs && 0 != facetQs.length) {
for (String q : facetQs) {
res.add(q, searcher.numDocs(qp.parse(q), docs));
Query qobj = QParser.getParser(q, null, req).getQuery();
res.add(q, searcher.numDocs(qobj, docs));
}
}

1
src/java/org/apache/solr/request/XMLWriter.java
View File

@ -473,6 +473,7 @@ final public class XMLWriter {
SolrIndexSearcher searcher = request.getSearcher();
DocIterator iterator = ids.iterator();
int sz = ids.size();
includeScore = includeScore && ids.hasScores();
for (int i=0; i<sz; i++) {
int id = iterator.nextDoc();
Document doc = searcher.doc(id, fields);

2
src/java/org/apache/solr/search/MissingStringLastComparatorSource.java
View File

@ -33,7 +33,7 @@ import java.io.IOException;
*/
// move to apache package and make public if it is accepted as a patch
class MissingStringLastComparatorSource implements SortComparatorSource {
public class MissingStringLastComparatorSource implements SortComparatorSource {
public static final String bigString="\uffff\uffff\uffff\uffff\uffff\uffff\uffff\uffffNULL_VAL";

2
src/java/org/apache/solr/search/QParser.java
View File

@ -197,7 +197,7 @@ public abstract class QParser {
QParserPlugin qplug = req.getCore().getQueryPlugin(type);
return qplug.createParser(qstr, localParams, req.getParams(), req);
}
}
}

11
src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -519,8 +519,15 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
private static Query matchAllDocsQuery = new MatchAllDocsQuery();
protected DocSet getDocSet(List<Query> queries) throws IOException {
/**
* Returns the set of document ids matching all queries.
* This method is cache-aware and attempts to retrieve the answer from the cache if possible.
* If the answer was not cached, it may have been inserted into the cache as a result of this call.
* This method can handle negative queries.
* <p>
* The DocSet returned should <b>not</b> be modified.
*/
public DocSet getDocSet(List<Query> queries) throws IOException {
if (queries==null) return null;
if (queries.size()==1) return getDocSet(queries.get(0));
DocSet answer=null;

13
src/java/org/apache/solr/search/SortSpec.java
View File

@ -18,6 +18,7 @@
package org.apache.solr.search;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
/***
* SortSpec encapsulates a Lucene Sort and a count of the number of documents
@ -44,6 +45,18 @@ public class SortSpec
sort = s;
}
public static boolean includesScore(Sort sort) {
if (sort==null) return true;
for (SortField sf : sort.getSort()) {
if (sf.getType() == SortField.SCORE) return true;
}
return false;
}
public boolean includesScore() {
return includesScore(sort);
}
/**
* Gets the Lucene Sort object, or null for the default sort
* by score descending.

2
src/java/org/apache/solr/search/Sorting.java
View File

@ -52,6 +52,6 @@ public class Sorting {
}
static final SortComparatorSource nullStringLastComparatorSource = new MissingStringLastComparatorSource();
static final SortComparatorSource nullStringLastComparatorSource = new MissingStringLastComparatorSource(null);
}

9
src/java/org/apache/solr/util/SolrPluginUtils.java
View File

@ -437,11 +437,12 @@ public class SolrPluginUtils {
Document doc = searcher.doc(id);
String strid = schema.printableUniqueKey(doc);
String docname = "";
if (strid != null) docname="id="+strid+",";
docname = docname + "internal_docid="+id;
explainList.add(docname, "\n" +explain.toString());
// String docname = "";
// if (strid != null) docname="id="+strid+",";
// docname = docname + "internal_docid="+id;
explainList.add(strid, "\n" +explain.toString());
}
return explainList;
}

4
src/test/org/apache/solr/BasicFunctionalityTest.java
View File

@ -225,7 +225,7 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase {
String resp = h.query(lrf.makeRequest("q", "text:hello", "debugQuery", "true"));
//System.out.println(resp);
// second doc ranked first
assertTrue( resp.indexOf("id=2") < resp.indexOf("id=1") );
assertTrue( resp.indexOf("\"2\"") < resp.indexOf("\"1\"") );
}
public void testFieldBoost() throws Exception {
@ -243,7 +243,7 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase {
String resp = h.query(lrf.makeRequest("q", "text:hello", "debugQuery", "true"));
//System.out.println(resp);
// second doc ranked first
assertTrue( resp.indexOf("id=2") < resp.indexOf("id=1") );
assertTrue( resp.indexOf("\"2\"") < resp.indexOf("\"1\"") );
}
public void testXMLWriter() throws Exception {

504
src/test/org/apache/solr/TestDistributedSearch.java Executable file
View File

@ -0,0 +1,504 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.AbstractSolrTestCase;
import java.io.File;
import java.util.*;
import junit.framework.TestCase;
/**
* TODO? perhaps use:
* http://docs.codehaus.org/display/JETTY/ServletTester
* rather then open a real connection?
*
* @version $Id$
* @since solr 1.3
*/
public class TestDistributedSearch extends TestCase {
Random r = new Random(0);
File testDir;
int controlPort = 8985;
SolrServer controlClient;
JettySolrRunner controlJetty;
int[] ports = new int[] {7574, 7576};
List<SolrServer> clients = new ArrayList<SolrServer>();
List<JettySolrRunner> jettys = new ArrayList<JettySolrRunner>();
String context = "/solr";
String shards;
String id="id";
String t1="a_t";
String i1="a_i";
@Override public void setUp() throws Exception
{
System.setProperty("solr.test.sys.prop1", "propone");
System.setProperty("solr.test.sys.prop2", "proptwo");
testDir = new File(System.getProperty("java.io.tmpdir")
+ System.getProperty("file.separator")
+ getClass().getName() + "-" + System.currentTimeMillis());
testDir.mkdirs();
}
@Override public void tearDown() throws Exception
{
destroyServers();
AbstractSolrTestCase.recurseDelete(testDir);
}
private void createServers() throws Exception {
controlJetty = createJetty(controlPort);
controlClient = createNewSolrServer(controlPort);
StringBuilder sb = new StringBuilder();
for (int port : ports) {
if (sb.length()>0) sb.append(',');
sb.append("localhost:"+port+context);
jettys.add(createJetty(port));
clients.add(createNewSolrServer(port));
}
shards = sb.toString();
}
private void destroyServers() throws Exception {
controlJetty.stop();
for (JettySolrRunner jetty : jettys) jetty.stop();
clients.clear();
jettys.clear();
}
private JettySolrRunner createJetty(int port) throws Exception {
File subDir = new File(testDir, ""+port);
subDir.mkdirs();
System.setProperty("solr.data.dir", subDir.toString());
JettySolrRunner jetty = new JettySolrRunner("/solr", port);
jetty.start();
return jetty;
}
protected SolrServer createNewSolrServer(int port)
{
try {
// setup the server...
String url = "http://localhost:"+port+context;
CommonsHttpSolrServer s = new CommonsHttpSolrServer( url );
s.setConnectionTimeout(100); // 1/10th sec
s.setDefaultMaxConnectionsPerHost(100);
s.setMaxTotalConnections(100);
return s;
}
catch( Exception ex ) {
throw new RuntimeException( ex );
}
}
void index(Object... fields) throws Exception {
SolrInputDocument doc = new SolrInputDocument();
for (int i=0; i<fields.length; i+=2) {
doc.addField((String)(fields[i]), fields[i+1]);
}
controlClient.add(doc);
int which = (doc.getField(id).toString().hashCode() &0x7fffffff) % clients.size();
SolrServer client = clients.get(which);
client.add(doc);
}
void index_specific(int serverNumber, Object... fields) throws Exception {
SolrInputDocument doc = new SolrInputDocument();
for (int i=0; i<fields.length; i+=2) {
doc.addField((String)(fields[i]), fields[i+1]);
}
controlClient.add(doc);
int which = serverNumber;
SolrServer client = clients.get(which);
client.add(doc);
}
void del(String q) throws Exception {
controlClient.deleteByQuery(q);
for (SolrServer client : clients) {
client.deleteByQuery(q);
}
}
// serial commit...
void commit() throws Exception {
controlClient.commit();
for (SolrServer client : clients) client.commit();
}
void query(Object... q) throws Exception {
ModifiableSolrParams params = new ModifiableSolrParams();
for (int i=0; i<q.length; i+=2) {
params.add(q[i].toString(), q[i+1].toString());
}
QueryResponse controlRsp = controlClient.query(params);
// query a random server
params.set("shards", shards);
int which = r.nextInt(clients.size());
SolrServer client = clients.get(which);
QueryResponse rsp = client.query(params);
compareResponses(rsp, controlRsp);
}
private static int ORDERED=1;
private static int SKIP=2;
private static int SKIPVAL=4;
private static int UNORDERED=8;
public static boolean eq(String a, String b) {
return a==b || (a != null && a.equals(b));
}
public static int flags(Map<String,Integer> handle, Object key) {
if (handle==null) return 0;
Integer f = handle.get(key);
return f == null ? 0 : f;
}
public static String compare(NamedList a, NamedList b, int flags, Map<String,Integer> handle) {
boolean ordered = (flags&UNORDERED) == 0;
int posa = 0, posb = 0;
int na = 0, nb = 0;
for(;;) {
if (posa >= a.size() || posb >= b.size()) {
break;
}
String namea=null, nameb=null;
Object vala=null, valb=null;
int flagsa, flagsb;
for (;;) {
namea = a.getName(posa);
vala = a.getVal(posa);
posa++;
flagsa = flags(handle, namea);
if ((flagsa & SKIP) != 0) continue;
na++;
break;
}
if (!ordered) posb=0; // reset if not ordered
while (posb<b.size()) {
nameb = b.getName(posb);
valb = b.getVal(posb);
posb++;
flagsb = flags(handle, nameb);
if ((flagsb & SKIP) != 0) continue;
if (eq(namea, nameb)) {
nb++;
break;
}
if (ordered) {
return "."+namea+"!="+nameb+" (unordered or missing)";
}
// if unordered, continue until we find the right field.
}
// ok, namea and nameb should be equal here already.
if ((flagsa & SKIPVAL) != 0) continue; // keys matching is enough
String cmp = compare(vala, valb, flagsa, handle);
if (cmp != null) return "."+namea+cmp;
}
if (na != nb) {
return ".size()=="+na+","+nb;
}
return null;
}
private static String compare1(Map a, Map b, int flags, Map<String,Integer> handle) {
String cmp;
for (Object keya : a.keySet()) {
Object vala = a.get(keya);
int flagsa = flags(handle, keya);
if ((flagsa & SKIP) != 0) continue;
if (!b.containsKey(keya)) {
return "[" + keya + "]==null";
}
if ((flagsa & SKIPVAL) != 0) continue;
Object valb = b.get(keya);
cmp = compare(vala, valb, flagsa, handle);
if (cmp != null) return "[" + keya + "]" + cmp;
}
return null;
}
public static String compare(Map a, Map b, int flags, Map<String,Integer> handle) {
String cmp;
cmp = compare1(a,b,flags,handle);
if (cmp != null) return cmp;
return compare1(b,a,flags,handle);
}
public static String compare(SolrDocument a, SolrDocument b, int flags, Map<String,Integer> handle) {
return compare(a.getFieldValuesMap(), b.getFieldValuesMap(), flags, handle);
}
public static String compare(SolrDocumentList a, SolrDocumentList b, int flags, Map<String,Integer> handle) {
boolean ordered = (flags & UNORDERED) == 0;
String cmp;
int f = flags(handle, "maxScore");
if ((f & SKIPVAL) == 0) {
cmp = compare(a.getMaxScore(), b.getMaxScore(), 0, handle);
if (cmp != null) return ".maxScore" + cmp;
} else {
if (a.getMaxScore() != null) {
if (b.getMaxScore() == null) {
return ".maxScore missing";
}
}
}
cmp = compare(a.getNumFound(), b.getNumFound(), 0, handle);
if (cmp != null) return ".numFound" + cmp;
cmp = compare(a.getStart(), b.getStart(), 0, handle);
if (cmp != null) return ".start" + cmp;
cmp = compare(a.size(), b.size(), 0, handle);
if (cmp != null) return ".size()" + cmp;
// only for completely ordered results (ties might be in a different order)
if (ordered) {
for (int i=0; i<a.size(); i++) {
cmp = compare(a.get(i), b.get(i), 0, handle);
if (cmp != null) return "["+i+"]"+cmp;
}
return null;
}
// unordered case
for (int i=0; i<a.size(); i++) {
SolrDocument doc = a.get(i);
Object key = doc.getFirstValue("id");
SolrDocument docb=null;
if (key==null) {
// no id field to correlate... must compare ordered
docb = b.get(i);
} else {
for (int j=0; j<b.size(); j++) {
docb = b.get(j);
if (key.equals(docb.getFirstValue("id"))) break;
}
}
// if (docb == null) return "[id="+key+"]";
cmp = compare(doc, docb, 0, handle);
if (cmp != null) return "[id="+key+"]" + cmp;
}
return null;
}
public static String compare(Object[] a, Object[] b, int flags, Map<String,Integer> handle) {
if (a.length != b.length) {
return ".length:"+a.length+"!="+b.length;
}
for (int i=0; i<a.length; i++) {
String cmp = compare(a[i], b[i], flags, handle);
if (cmp != null) return "["+i+"]"+cmp;
}
return null;
}
static String compare(Object a, Object b, int flags, Map<String,Integer> handle) {
if (a==b) return null;
if (a==null || b==null) return ":" +a + "!=" + b;
if (a instanceof NamedList && b instanceof NamedList) {
return compare((NamedList)a, (NamedList)b, flags, handle);
}
if (a instanceof SolrDocumentList && b instanceof SolrDocumentList) {
return compare((SolrDocumentList)a, (SolrDocumentList)b, flags, handle);
}
if (a instanceof SolrDocument && b instanceof SolrDocument) {
return compare((SolrDocument)a, (SolrDocument)b, flags, handle);
}
if (a instanceof Map && b instanceof Map) {
return compare((Map)a, (Map)b, flags, handle);
}
if (a instanceof Object[] && b instanceof Object[]) {
return compare((Object[])a, (Object[])b, flags, handle);
}
if (!(a.equals(b))) {
return ":" + a + "!=" + b;
}
return null;
}
void compareResponses(QueryResponse a, QueryResponse b) {
String cmp;
System.out.println(a);
System.out.println(b);
cmp = compare(a.getResponse(), b.getResponse(), flags, handle);
if (cmp != null) {
System.out.println(a);
System.out.println(b);
TestCase.fail(cmp);
}
}
int flags;
Map<String, Integer> handle = new HashMap<String,Integer>();
public void testDistribSearch() throws Exception {
for (int nServers=1; nServers<4; nServers++) {
ports = new int[nServers];
for (int i=0; i<nServers; i++) {
ports[i] = 7574 + i*2;
}
doTest();
}
}
public void doTest() throws Exception {
createServers();
del("*:*");
index(id,1, i1, 100,t1,"now is the time for all good men");
index(id,2, i1, 50 ,t1,"to come to the aid of their country.");
index(id,3, i1, 2 ,t1,"how now brown cow");
index(id,4, i1, -100 ,t1,"the quick fox jumped over the lazy dog");
index(id,5, i1, 500 ,t1,"the quick fox jumped way over the lazy dog");
index(id,6, i1, -600 ,t1,"humpty dumpy sat on a wall");
index(id,7, i1, 123 ,t1,"humpty dumpy had a great fall");
index(id,8, i1, 876 ,t1,"all the kings horses and all the kings men");
index(id,9, i1, 7 ,t1,"couldn't put humpty together again");
index(id,10, i1, 4321 ,t1,"this too shal pass");
index(id,11, i1, -987 ,t1,"An eye for eye only ends up making the whole world blind.");
index(id,12, i1, 379 ,t1,"Great works are performed, not by strength, but by perseverance.");
commit();
handle.clear();
handle.put("QTime", SKIPVAL);
handle.put("timestamp", SKIPVAL);
// these queries should be exactly ordered and scores should exactly match
query("q","*:*", "sort",i1+" desc");
query("q","<!func>"+i1);
query("q","<!func>"+i1, "fl","*,score"); // even scores should match exactly here
handle.put("highlighting", UNORDERED);
handle.put("response", UNORDERED);
query("q","quick");
query("q","all","fl","id","start","0");
query("q","all","fl","foofoofoo","start","0"); // no fields in returned docs
query("q","all","fl","id","start","100");
handle.put("maxScore", SKIPVAL);
handle.put("score", SKIPVAL);
query("q","quick","fl","*,score");
query("q","all","fl","*,score","start","1");
query("q","all","fl","*,score","start","100");
query("q","now their fox sat had put","fl","*,score",
"hl","true","hl.fl",t1);
query("q","now their fox sat had put","fl","foofoofoo",
"hl","true","hl.fl",t1);
handle.put("debug", UNORDERED);
handle.put("time", SKIPVAL);
query("q","now their fox sat had put","fl","*,score",
"debugQuery", "true");
query("q","*:*", "rows",100, "facet","true", "facet.field",t1);
query("q","*:*", "rows",100, "facet","true", "facet.query","quick", "facet.query","all", "facet.query","*:*");
query("q","*:*", "rows",100, "facet","true", "facet.field",t1, "facet.offset",1);
query("q","*:*", "rows",100, "facet","true", "facet.field",t1,"facet.mincount",2);
// index the same document to two servers and make sure things
// don't blow up.
if (clients.size()>=2) {
index(id,100, i1, 107 ,t1,"oh no, a duplicate!");
for (int i=0; i<clients.size(); i++) {
index_specific(i, id,100, i1, 107 ,t1,"oh no, a duplicate!");
}
commit();
query("q","duplicate", "hl","true", "hl.fl", t1);
query("q","fox duplicate horses", "hl","true", "hl.fl", t1);
query("q","*:*", "rows",100);
}
// Thread.sleep(10000000000L);
destroyServers();
}
}

4
src/test/test-files/solr/conf/solrconfig.xml
View File

@ -27,9 +27,7 @@
<!-- Used to specify an alternate directory to hold all index data.
It defaults to "index" if not present, and should probably
not be changed if replication is in use. -->
<!--
<indexDir>index</indexDir>
-->
<dataDir>${solr.data.dir:./solr/data}</dataDir>
<indexDefaults>
<!-- Values here affect all index writers and act as a default