mirror of https://github.com/apache/lucene.git
SOLR-303 Distributed Search over HTTP
git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@631357 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
0c2e3ebdd4
commit
cc61bb647b
|
@ -199,6 +199,15 @@ New Features
|
|||
with "304 Not Modified" when appropriate. New options have been added
|
||||
to solrconfig.xml to influence this behavior.
|
||||
(Thomas Peuss via hossman)
|
||||
|
||||
40. SOLR-303: Distributed Search over HTTP. Specification of shards
|
||||
argument causes Solr to query those shards and merge the results
|
||||
into a single response. Querying, field faceting (sorted only),
|
||||
query faceting, highlighting, and debug information are supported
|
||||
in distributed mode.
|
||||
(Sharad Agarwal, Patrick O'Leary, Sabyasachi Dalal, Stu Hood,
|
||||
ryan, yonik)
|
||||
|
||||
|
||||
Changes in runtime behavior
|
||||
|
||||
|
|
|
@ -235,9 +235,10 @@
|
|||
depends="compile-common,init-forrest-entities">
|
||||
|
||||
<solr-javac destdir="${dest}/core"
|
||||
classpathref="compile.classpath">
|
||||
classpathref="compile.classpath.solrj-embedded">
|
||||
<src path="${src}/java" />
|
||||
<src path="${src}/webapp/src" />
|
||||
<src path="client/java/solrj/src" />
|
||||
<exclude name="org/apache/solr/common/**" />
|
||||
</solr-javac>
|
||||
</target>
|
||||
|
@ -628,6 +629,7 @@
|
|||
<include name="${fullnamever}.jar" />
|
||||
<include name="${fullname}-common-${version}.jar" />
|
||||
</lib>
|
||||
<lib dir="client/java/solrj/lib"/>
|
||||
<fileset dir="${src}/webapp/resources" />
|
||||
<metainf dir="${basedir}" includes="LICENSE.txt,NOTICE.txt"/>
|
||||
</war>
|
||||
|
|
|
@ -51,7 +51,6 @@ public class QueryResponse extends SolrResponseBase
|
|||
|
||||
// Debug Info
|
||||
private Map<String,Object> _debugMap = null;
|
||||
private Map<String,Integer> _docIdMap = null;
|
||||
private Map<String,String> _explainMap = null;
|
||||
|
||||
public QueryResponse( NamedList<Object> res )
|
||||
|
@ -81,9 +80,7 @@ public class QueryResponse extends SolrResponseBase
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static final String DKEY = ",internal_docid=";
|
||||
|
||||
|
||||
private void extractDebugInfo( NamedList<Object> debug )
|
||||
{
|
||||
_debugMap = new LinkedHashMap<String, Object>(); // keep the order
|
||||
|
@ -93,20 +90,11 @@ public class QueryResponse extends SolrResponseBase
|
|||
|
||||
// Parse out interisting bits from the debug info
|
||||
_explainMap = new HashMap<String, String>();
|
||||
_docIdMap = new HashMap<String, Integer>();
|
||||
NamedList<String> explain = (NamedList<String>)_debugMap.get( "explain" );
|
||||
if( explain != null ) {
|
||||
for( Map.Entry<String, String> info : explain ) {
|
||||
String key = info.getKey();
|
||||
int idx0 = key.indexOf( '=' )+1;
|
||||
int idx1 = info.getKey().indexOf( DKEY );
|
||||
int idx2 = idx1 + DKEY.length();
|
||||
|
||||
String id = key.substring( idx0, idx1 );
|
||||
String docID = key.substring( idx2 );
|
||||
|
||||
_explainMap.put( id, info.getValue() );
|
||||
_docIdMap.put( id, Integer.valueOf( docID ) );
|
||||
_explainMap.put( key, info.getValue() );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -182,10 +170,6 @@ public class QueryResponse extends SolrResponseBase
|
|||
return _debugMap;
|
||||
}
|
||||
|
||||
public Map<String, Integer> getDocIdMap() {
|
||||
return _docIdMap;
|
||||
}
|
||||
|
||||
public Map<String, String> getExplainMap() {
|
||||
return _explainMap;
|
||||
}
|
||||
|
|
|
@ -25,7 +25,7 @@ import org.apache.solr.common.util.NamedList;
|
|||
* @version $Id$
|
||||
* @since solr 1.3
|
||||
*/
|
||||
public abstract class SolrResponseBase extends SolrResponse
|
||||
public class SolrResponseBase extends SolrResponse
|
||||
{
|
||||
private long elapsedTime = -1;
|
||||
private NamedList<Object> response = null;
|
||||
|
|
|
@ -118,6 +118,27 @@ public class StrUtils {
|
|||
return lst;
|
||||
}
|
||||
|
||||
/** Creates a backslash escaped string, joining all the items. */
|
||||
public static String join(List<String> items, char separator) {
|
||||
StringBuilder sb = new StringBuilder(items.size() << 3);
|
||||
boolean first=true;
|
||||
for (String item : items) {
|
||||
if (first) {
|
||||
first = false;
|
||||
} else {
|
||||
sb.append(separator);
|
||||
}
|
||||
for (int i=0; i<item.length(); i++) {
|
||||
char ch = item.charAt(i);
|
||||
if (ch=='\\' || ch == separator) {
|
||||
sb.append('\\');
|
||||
}
|
||||
sb.append(ch);
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static List<String> splitWS(String s, boolean decode) {
|
||||
|
|
|
@ -179,7 +179,7 @@ public class MoreLikeThisHandler extends RequestHandlerBase
|
|||
rsp.add( "facet_counts", null );
|
||||
}
|
||||
else {
|
||||
SimpleFacets f = new SimpleFacets(searcher, mltDocs.docSet, params );
|
||||
SimpleFacets f = new SimpleFacets(req, mltDocs.docSet, params );
|
||||
rsp.add( "facet_counts", f.getFacetCounts() );
|
||||
}
|
||||
}
|
||||
|
|
|
@ -27,6 +27,7 @@ import org.apache.solr.request.SolrQueryRequest;
|
|||
import org.apache.solr.request.SolrQueryResponse;
|
||||
import org.apache.solr.request.SolrRequestHandler;
|
||||
import org.apache.solr.util.SolrPluginUtils;
|
||||
import org.apache.lucene.queryParser.ParseException;
|
||||
|
||||
import java.net.URL;
|
||||
|
||||
|
@ -117,6 +118,9 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
|
|||
handleRequestBody( req, rsp );
|
||||
} catch (Exception e) {
|
||||
SolrException.log(SolrCore.log,e);
|
||||
if (e instanceof ParseException) {
|
||||
e = new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
|
||||
}
|
||||
rsp.setException(e);
|
||||
numErrors++;
|
||||
}
|
||||
|
|
|
@ -17,11 +17,12 @@
|
|||
|
||||
package org.apache.solr.handler;
|
||||
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.handler.component.*;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
|
||||
import org.apache.solr.handler.component.SearchHandler;
|
||||
|
||||
/**
|
||||
* @version $Id$
|
||||
*
|
||||
|
|
|
@ -18,21 +18,20 @@
|
|||
package org.apache.solr.handler.component;
|
||||
|
||||
import static org.apache.solr.common.params.CommonParams.FQ;
|
||||
import org.apache.solr.common.params.HighlightParams;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryResponse;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.search.QueryParsing;
|
||||
import org.apache.solr.util.SolrPluginUtils;
|
||||
|
||||
/**
|
||||
* TODO!
|
||||
* Adds debugging information to a request.
|
||||
*
|
||||
* @version $Id$
|
||||
* @since solr 1.3
|
||||
|
@ -42,48 +41,182 @@ public class DebugComponent extends SearchComponent
|
|||
public static final String COMPONENT_NAME = "debug";
|
||||
|
||||
@Override
|
||||
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
|
||||
public void prepare(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
|
||||
public void process(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
|
||||
if( builder.isDebug() ) {
|
||||
NamedList stdinfo = SolrPluginUtils.doStandardDebug( req,
|
||||
builder.getQueryString(), builder.getQuery(), builder.getResults().docList);
|
||||
if( rb.isDebug() ) {
|
||||
NamedList stdinfo = SolrPluginUtils.doStandardDebug( rb.req,
|
||||
rb.getQueryString(), rb.getQuery(), rb.getResults().docList);
|
||||
|
||||
NamedList info = builder.getDebugInfo();
|
||||
NamedList info = rb.getDebugInfo();
|
||||
if( info == null ) {
|
||||
builder.setDebugInfo( stdinfo );
|
||||
rb.setDebugInfo( stdinfo );
|
||||
info = stdinfo;
|
||||
}
|
||||
else {
|
||||
info.addAll( stdinfo );
|
||||
}
|
||||
|
||||
if (builder.getQparser() != null) {
|
||||
builder.getQparser().addDebugInfo(builder.getDebugInfo());
|
||||
if (rb.getQparser() != null) {
|
||||
rb.getQparser().addDebugInfo(rb.getDebugInfo());
|
||||
}
|
||||
|
||||
if (null != builder.getDebugInfo() ) {
|
||||
if (null != builder.getFilters() ) {
|
||||
info.add("filter_queries",req.getParams().getParams(FQ));
|
||||
List<String> fqs = new ArrayList<String>(builder.getFilters().size());
|
||||
for (Query fq : builder.getFilters()) {
|
||||
fqs.add(QueryParsing.toString(fq, req.getSchema()));
|
||||
if (null != rb.getDebugInfo() ) {
|
||||
if (null != rb.getFilters() ) {
|
||||
info.add("filter_queries",rb.req.getParams().getParams(FQ));
|
||||
List<String> fqs = new ArrayList<String>(rb.getFilters().size());
|
||||
for (Query fq : rb.getFilters()) {
|
||||
fqs.add(QueryParsing.toString(fq, rb.req.getSchema()));
|
||||
}
|
||||
info.add("parsed_filter_queries",fqs);
|
||||
}
|
||||
|
||||
// Add this directly here?
|
||||
rsp.add("debug", builder.getDebugInfo() );
|
||||
rb.rsp.add("debug", rb.getDebugInfo() );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
|
||||
if (!rb.isDebug()) return;
|
||||
|
||||
// Turn on debug to get explain only only when retrieving fields
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
|
||||
sreq.purpose |= ShardRequest.PURPOSE_GET_DEBUG;
|
||||
sreq.params.set("debugQuery", "true");
|
||||
} else {
|
||||
sreq.params.set("debugQuery", "false");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
|
||||
}
|
||||
|
||||
private Set<String> excludeSet = new HashSet<String>(Arrays.asList("explain"));
|
||||
|
||||
@Override
|
||||
public void finishStage(ResponseBuilder rb) {
|
||||
if (rb.isDebug() && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
|
||||
NamedList info = null;
|
||||
NamedList explain = new SimpleOrderedMap();
|
||||
|
||||
Object[] arr = new Object[rb.resultIds.size() * 2];
|
||||
|
||||
for (ShardRequest sreq : rb.finished) {
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_DEBUG) == 0) continue;
|
||||
for (ShardResponse srsp : sreq.responses) {
|
||||
NamedList sdebug = (NamedList)srsp.rsp.getResponse().get("debug");
|
||||
info = (NamedList)merge(sdebug, info, excludeSet);
|
||||
|
||||
NamedList sexplain = (NamedList)sdebug.get("explain");
|
||||
|
||||
for (int i=0; i<sexplain.size(); i++) {
|
||||
String id = sexplain.getName(i);
|
||||
// TODO: lookup won't work for non-string ids... String vs Float
|
||||
ShardDoc sdoc = rb.resultIds.get(id);
|
||||
int idx = sdoc.positionInResponse;
|
||||
arr[idx<<1] = id;
|
||||
arr[(idx<<1)+1] = sexplain.getVal(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
explain = HighlightComponent.removeNulls(new NamedList(Arrays.asList(arr)));
|
||||
int idx = info.indexOf("explain",0);
|
||||
if (idx>=0) {
|
||||
info.setVal(idx, explain);
|
||||
} else {
|
||||
info.add("explain", explain);
|
||||
}
|
||||
|
||||
rb.setDebugInfo(info);
|
||||
rb.rsp.add("debug", rb.getDebugInfo() );
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Object merge(Object source, Object dest, Set<String> exclude) {
|
||||
if (source == null) return dest;
|
||||
if (dest == null) {
|
||||
if (source instanceof NamedList) {
|
||||
dest = source instanceof SimpleOrderedMap ? new SimpleOrderedMap() : new NamedList();
|
||||
} else {
|
||||
return source;
|
||||
}
|
||||
} else {
|
||||
|
||||
if (dest instanceof Collection) {
|
||||
if (source instanceof Collection) {
|
||||
((Collection)dest).addAll((Collection)source);
|
||||
} else {
|
||||
((Collection)dest).add(source);
|
||||
}
|
||||
return dest;
|
||||
} else if (source instanceof Number) {
|
||||
if (dest instanceof Number) {
|
||||
if (source instanceof Double || dest instanceof Double) {
|
||||
return ((Number)source).doubleValue() + ((Number)dest).doubleValue();
|
||||
}
|
||||
return ((Number)source).longValue() + ((Number)dest).longValue();
|
||||
}
|
||||
// fall through
|
||||
} else if (source instanceof String) {
|
||||
if (source.equals(dest)) {
|
||||
return dest;
|
||||
}
|
||||
// fall through
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (source instanceof NamedList && dest instanceof NamedList) {
|
||||
NamedList tmp = new NamedList();
|
||||
NamedList sl = (NamedList)source;
|
||||
NamedList dl = (NamedList)dest;
|
||||
for (int i=0; i<sl.size(); i++) {
|
||||
String skey = sl.getName(i);
|
||||
if (exclude != null && exclude.contains(skey)) continue;
|
||||
Object sval = sl.getVal(i);
|
||||
int didx = -1;
|
||||
|
||||
// optimize case where elements are in same position
|
||||
if (i < dl.size()) {
|
||||
String dkey = dl.getName(i);
|
||||
if (skey == dkey || (skey!=null && skey.equals(dkey))) {
|
||||
didx = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (didx == -1) {
|
||||
didx = dl.indexOf(skey, 0);
|
||||
}
|
||||
|
||||
if (didx == -1) {
|
||||
tmp.add(skey, merge(sval, null, null));
|
||||
} else {
|
||||
dl.setVal(didx, merge(sval, dl.getVal(didx), null));
|
||||
}
|
||||
}
|
||||
dl.addAll(tmp);
|
||||
return dl;
|
||||
}
|
||||
|
||||
// merge unlike elements in a list
|
||||
List t = new ArrayList();
|
||||
t.add(dest);
|
||||
t.add(source);
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/////////////////////////////////////////////
|
||||
/// SolrInfoMBean
|
||||
|
|
|
@ -19,12 +19,19 @@ package org.apache.solr.handler.component;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.solr.common.params.FacetParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.request.SimpleFacets;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryResponse;
|
||||
import org.apache.solr.util.OpenBitSet;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.search.QueryParsing;
|
||||
import org.apache.lucene.queryParser.ParseException;
|
||||
|
||||
/**
|
||||
* TODO!
|
||||
|
@ -37,34 +44,336 @@ public class FacetComponent extends SearchComponent
|
|||
public static final String COMPONENT_NAME = "facet";
|
||||
|
||||
@Override
|
||||
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
|
||||
public void prepare(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
SolrParams params = req.getParams();
|
||||
if (params.getBool(FacetParams.FACET,false)) {
|
||||
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
|
||||
builder.setNeedDocSet( true );
|
||||
if (rb.req.getParams().getBool(FacetParams.FACET,false)) {
|
||||
rb.setNeedDocSet( true );
|
||||
rb.doFacets = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Actually run the query
|
||||
* @param rb
|
||||
*/
|
||||
@Override
|
||||
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
|
||||
public void process(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
SolrParams params = req.getParams();
|
||||
if (params.getBool(FacetParams.FACET,false)) {
|
||||
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
|
||||
|
||||
SimpleFacets f = new SimpleFacets(req.getSearcher(),
|
||||
builder.getResults().docSet,
|
||||
params );
|
||||
if (rb.doFacets) {
|
||||
SolrParams params = rb.req.getParams();
|
||||
SimpleFacets f = new SimpleFacets(rb.req,
|
||||
rb.getResults().docSet,
|
||||
params );
|
||||
|
||||
// TODO ???? add this directly to the response?
|
||||
rsp.add( "facet_counts", f.getFacetCounts() );
|
||||
// TODO ???? add this directly to the response, or to the builder?
|
||||
rb.rsp.add( "facet_counts", f.getFacetCounts() );
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int distributedProcess(ResponseBuilder rb) throws IOException {
|
||||
if (!rb.doFacets) {
|
||||
return ResponseBuilder.STAGE_DONE;
|
||||
}
|
||||
|
||||
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
|
||||
// overlap facet refinement requests (those shards that we need a count for
|
||||
// particular facet values from), where possible, with
|
||||
// the requests to get fields (because we know that is the
|
||||
// only other required phase).
|
||||
// We do this in distributedProcess so we can look at all of the
|
||||
// requests in the outgoing queue at once.
|
||||
|
||||
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
|
||||
List<String> fqueries = rb._facetInfo._toRefine[shardNum];
|
||||
if (fqueries == null || fqueries.size()==0) continue;
|
||||
|
||||
String shard = rb.shards[shardNum];
|
||||
|
||||
ShardRequest refine = null;
|
||||
boolean newRequest = false;
|
||||
|
||||
// try to find a request that is already going out to that shard.
|
||||
// If nshards becomes to great, we way want to move to hashing for better
|
||||
// scalability.
|
||||
for (ShardRequest sreq : rb.outgoing) {
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS)!=0
|
||||
&& sreq.shards != null & sreq.shards.length==1
|
||||
&& sreq.shards[0].equals(shard))
|
||||
{
|
||||
refine = sreq;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (refine == null) {
|
||||
// we didn't find any other suitable requests going out to that shard, so
|
||||
// create one ourselves.
|
||||
newRequest = true;
|
||||
refine = new ShardRequest();
|
||||
refine.shards = new String[]{rb.shards[shardNum]};
|
||||
refine.params = new ModifiableSolrParams(rb.req.getParams());
|
||||
// don't request any documents
|
||||
refine.params.remove("start");
|
||||
refine.params.set("rows","0");
|
||||
}
|
||||
|
||||
refine.purpose |= ShardRequest.PURPOSE_REFINE_FACETS;
|
||||
refine.params.set(FacetParams.FACET,"true");
|
||||
refine.params.remove(FacetParams.FACET_FIELD);
|
||||
// TODO: perhaps create a more compact facet.terms method?
|
||||
refine.params.set(FacetParams.FACET_QUERY, fqueries.toArray(new String[fqueries.size()]));
|
||||
|
||||
if (newRequest) {
|
||||
rb.addRequest(this, refine);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ResponseBuilder.STAGE_DONE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
|
||||
if (!rb.doFacets) return;
|
||||
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
|
||||
sreq.purpose |= ShardRequest.PURPOSE_GET_FACETS;
|
||||
|
||||
FacetInfo fi = rb._facetInfo;
|
||||
if (fi == null) {
|
||||
rb._facetInfo = fi = new FacetInfo();
|
||||
fi.parse(rb.req.getParams(), rb);
|
||||
// should already be true...
|
||||
// sreq.params.set(FacetParams.FACET, "true");
|
||||
}
|
||||
|
||||
sreq.params.remove(FacetParams.FACET_MINCOUNT);
|
||||
sreq.params.remove(FacetParams.FACET_OFFSET);
|
||||
sreq.params.remove(FacetParams.FACET_LIMIT);
|
||||
|
||||
for (DistribFieldFacet dff : fi.topFacets.values()) {
|
||||
String paramStart = "f." + dff.field + '.';
|
||||
sreq.params.remove(paramStart + FacetParams.FACET_MINCOUNT);
|
||||
sreq.params.remove(paramStart + FacetParams.FACET_OFFSET);
|
||||
|
||||
// set the initial limit higher in increase accuracy
|
||||
dff.initialLimit = dff.offset + dff.limit;
|
||||
dff.initialLimit = (int)(dff.initialLimit * 1.5) + 10;
|
||||
|
||||
// Uncomment the following line when testing to supress over-requesting facets and
|
||||
// thus cause more facet refinement queries.
|
||||
// dff.initialLimit = dff.offset + dff.limit;
|
||||
|
||||
sreq.params.set(paramStart + FacetParams.FACET_LIMIT, dff.initialLimit);
|
||||
}
|
||||
} else {
|
||||
// turn off faceting on other requests
|
||||
sreq.params.set(FacetParams.FACET, "false");
|
||||
// we could optionally remove faceting params
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
|
||||
if (!rb.doFacets) return;
|
||||
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FACETS)!=0) {
|
||||
countFacets(rb, sreq);
|
||||
} else if ((sreq.purpose & ShardRequest.PURPOSE_REFINE_FACETS)!=0) {
|
||||
refineFacets(rb, sreq);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
private void countFacets(ResponseBuilder rb, ShardRequest sreq) {
|
||||
FacetInfo fi = rb._facetInfo;
|
||||
|
||||
for (ShardResponse srsp: sreq.responses) {
|
||||
int shardNum = rb.getShardNum(srsp.shard);
|
||||
NamedList facet_counts = (NamedList)srsp.rsp.getResponse().get("facet_counts");
|
||||
|
||||
// handle facet queries
|
||||
NamedList facet_queries = (NamedList)facet_counts.get("facet_queries");
|
||||
if (facet_queries != null) {
|
||||
for (int i=0; i<facet_queries.size(); i++) {
|
||||
String facet_q = (String)facet_queries.getName(i);
|
||||
long count = ((Number)facet_queries.getVal(i)).longValue();
|
||||
Long prevCount = fi.queryFacets.get(facet_q);
|
||||
if (prevCount != null) count += prevCount;
|
||||
fi.queryFacets.put(facet_q, count);
|
||||
}
|
||||
}
|
||||
|
||||
// step through each facet.field, adding results from this shard
|
||||
NamedList facet_fields = (NamedList)facet_counts.get("facet_fields");
|
||||
for (DistribFieldFacet dff : fi.topFacets.values()) {
|
||||
dff.add(shardNum, (NamedList)facet_fields.get(dff.field), dff.initialLimit);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// This code currently assumes that there will be only a single
|
||||
// request ((with responses from all shards) sent out to get facets...
|
||||
// otherwise we would need to wait until all facet responses were received.
|
||||
//
|
||||
|
||||
// list of queries to send each shard
|
||||
List<String>[] toRefine = new List[rb.shards.length];
|
||||
fi._toRefine = toRefine;
|
||||
for (int i=0; i<toRefine.length; i++) {
|
||||
toRefine[i] = new ArrayList<String>();
|
||||
}
|
||||
|
||||
|
||||
for (DistribFieldFacet dff : fi.topFacets.values()) {
|
||||
ShardFacetCount[] counts = dff.getSorted();
|
||||
int ntop = Math.min(counts.length, dff.offset + dff.limit);
|
||||
long smallestCount = counts[ntop-1].count;
|
||||
|
||||
for (int i=0; i<counts.length; i++) {
|
||||
ShardFacetCount sfc = counts[i];
|
||||
String query = null;
|
||||
boolean needRefinement = false;
|
||||
|
||||
if (i<ntop) {
|
||||
// automatically flag the top values for refinement
|
||||
needRefinement = true;
|
||||
} else {
|
||||
// calculate the maximum value that this term may have
|
||||
// and if it is >= smallestCount, then flag for refinement
|
||||
long maxCount = sfc.count;
|
||||
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
|
||||
OpenBitSet obs = dff.counted[shardNum];
|
||||
if (!obs.get(sfc.termNum)) {
|
||||
// if missing from this shard, add the max it could be
|
||||
maxCount += dff.maxPossible(sfc,shardNum);
|
||||
}
|
||||
}
|
||||
if (maxCount >= smallestCount) {
|
||||
// TODO: on a tie, we could check the term values
|
||||
needRefinement = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (needRefinement) {
|
||||
// add a query for each shard missing the term that needs refinement
|
||||
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
|
||||
OpenBitSet obs = dff.counted[shardNum];
|
||||
if (!obs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
|
||||
dff.needRefinements = true;
|
||||
if (query==null) query = dff.makeQuery(sfc);
|
||||
toRefine[shardNum].add(query);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void refineFacets(ResponseBuilder rb, ShardRequest sreq) {
|
||||
FacetInfo fi = rb._facetInfo;
|
||||
|
||||
for (ShardResponse srsp: sreq.responses) {
|
||||
int shardNum = rb.getShardNum(srsp.shard);
|
||||
NamedList facet_counts = (NamedList)srsp.rsp.getResponse().get("facet_counts");
|
||||
NamedList facet_queries = (NamedList)facet_counts.get("facet_queries");
|
||||
|
||||
// These are single term queries used to fill in missing counts
|
||||
// for facet.field queries
|
||||
for (int i=0; i<facet_queries.size(); i++) {
|
||||
try {
|
||||
|
||||
String facet_q = (String)facet_queries.getName(i);
|
||||
long count = ((Number)facet_queries.getVal(i)).longValue();
|
||||
|
||||
// expect <!field f=field>value style params
|
||||
SolrParams qparams = QueryParsing.getLocalParams(facet_q,null);
|
||||
String field = qparams.get(QueryParsing.F);
|
||||
String val = qparams.get(QueryParsing.V);
|
||||
|
||||
// Find the right field.facet for this field
|
||||
DistribFieldFacet dff = fi.topFacets.get(field);
|
||||
// Find the right constraint count for this value
|
||||
ShardFacetCount sfc = dff.counts.get(val);
|
||||
// TODO REMOVE
|
||||
System.out.println("Got " + facet_q + " , refining count: " + sfc + " += " + count);
|
||||
|
||||
sfc.count += count;
|
||||
|
||||
} catch (ParseException e) {
|
||||
// shouldn't happen, so fail for now rather than covering it up
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finishStage(ResponseBuilder rb) {
|
||||
if (!rb.doFacets || rb.stage != ResponseBuilder.STAGE_GET_FIELDS) return;
|
||||
// wait until STAGE_GET_FIELDS
|
||||
// so that "result" is already stored in the response (for aesthetics)
|
||||
|
||||
|
||||
FacetInfo fi = rb._facetInfo;
|
||||
|
||||
NamedList facet_counts = new NamedList();
|
||||
NamedList facet_queries = new NamedList();
|
||||
facet_counts.add("facet_queries",facet_queries);
|
||||
for (Map.Entry<String,Long> entry : fi.queryFacets.entrySet()) {
|
||||
facet_queries.add(entry.getKey(), num(entry.getValue()));
|
||||
}
|
||||
|
||||
NamedList facet_fields = new NamedList();
|
||||
facet_counts.add("facet_fields", facet_fields);
|
||||
|
||||
for (DistribFieldFacet dff : fi.topFacets.values()) {
|
||||
SimpleOrderedMap fieldCounts = new SimpleOrderedMap();
|
||||
facet_fields.add(dff.field, fieldCounts);
|
||||
|
||||
ShardFacetCount[] counts = dff.countSorted;
|
||||
if (dff.needRefinements) {
|
||||
counts = dff.getSorted();
|
||||
}
|
||||
|
||||
int end = Math.min(dff.offset + dff.limit, counts.length);
|
||||
for (int i=dff.offset; i<end; i++) {
|
||||
if (counts[i].count < dff.minCount) break;
|
||||
fieldCounts.add(counts[i].name, num(counts[i].count));
|
||||
}
|
||||
|
||||
if (dff.missing) {
|
||||
fieldCounts.add(null, num(dff.missingCount));
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: list facets (sorted by natural order)
|
||||
// TODO: facet dates
|
||||
facet_counts.add("facet_dates", new NamedList());
|
||||
|
||||
rb.rsp.add("facet_counts", facet_counts);
|
||||
|
||||
rb._facetInfo = null; // could be big, so release asap
|
||||
}
|
||||
|
||||
|
||||
// use <int> tags for smaller facet counts (better back compatibility)
|
||||
private Number num(long val) {
|
||||
if (val < Integer.MAX_VALUE) return (int)val;
|
||||
else return val;
|
||||
}
|
||||
private Number num(Long val) {
|
||||
if (val.longValue() < Integer.MAX_VALUE) return val.intValue();
|
||||
else return val;
|
||||
}
|
||||
|
||||
|
||||
/////////////////////////////////////////////
|
||||
/// SolrInfoMBean
|
||||
////////////////////////////////////////////
|
||||
|
@ -94,3 +403,171 @@ public class FacetComponent extends SearchComponent
|
|||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
class FacetInfo {
|
||||
List<String>[] _toRefine;
|
||||
|
||||
void parse(SolrParams params, ResponseBuilder rb) {
|
||||
queryFacets = new LinkedHashMap<String,Long>();
|
||||
topFacets = new LinkedHashMap<String,DistribFieldFacet>();
|
||||
listFacets = new LinkedHashMap<String,DistribFieldFacet>();
|
||||
|
||||
String[] facetQs = params.getParams(FacetParams.FACET_QUERY);
|
||||
if (facetQs != null) {
|
||||
for (String query : facetQs) {
|
||||
queryFacets.put(query,0L);
|
||||
}
|
||||
}
|
||||
|
||||
String[] facetFs = params.getParams(FacetParams.FACET_FIELD);
|
||||
if (facetFs != null) {
|
||||
for (String field : facetFs) {
|
||||
DistribFieldFacet ff = new DistribFieldFacet(rb, field);
|
||||
ff.fillParams(params, field);
|
||||
if (ff.sort) {
|
||||
topFacets.put(field, ff);
|
||||
} else {
|
||||
listFacets.put(field, ff);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LinkedHashMap<String,Long> queryFacets;
|
||||
LinkedHashMap<String,DistribFieldFacet> topFacets; // field facets that order by constraint count (sort=true)
|
||||
LinkedHashMap<String,DistribFieldFacet> listFacets; // field facets that list values in term order
|
||||
}
|
||||
|
||||
|
||||
class FieldFacet {
|
||||
String field;
|
||||
int offset;
|
||||
int limit;
|
||||
int minCount;
|
||||
boolean sort;
|
||||
boolean missing;
|
||||
String prefix;
|
||||
long missingCount;
|
||||
|
||||
void fillParams(SolrParams params, String field) {
|
||||
this.field = field;
|
||||
this.offset = params.getFieldInt(field, FacetParams.FACET_OFFSET, 0);
|
||||
this.limit = params.getFieldInt(field, FacetParams.FACET_LIMIT, 100);
|
||||
Integer mincount = params.getFieldInt(field, FacetParams.FACET_MINCOUNT);
|
||||
if (mincount==null) {
|
||||
Boolean zeros = params.getFieldBool(field, FacetParams.FACET_ZEROS);
|
||||
// mincount = (zeros!=null && zeros) ? 0 : 1;
|
||||
mincount = (zeros!=null && !zeros) ? 1 : 0;
|
||||
// current default is to include zeros.
|
||||
}
|
||||
this.minCount = mincount;
|
||||
this.missing = params.getFieldBool(field, FacetParams.FACET_MISSING, false);
|
||||
// default to sorting if there is a limit.
|
||||
this.sort = params.getFieldBool(field, FacetParams.FACET_SORT, limit>0);
|
||||
this.prefix = params.getFieldParam(field,FacetParams.FACET_PREFIX);
|
||||
}
|
||||
}
|
||||
|
||||
class DistribFieldFacet extends FieldFacet {
|
||||
SchemaField sf;
|
||||
|
||||
// the max possible count for a term appearing on no list
|
||||
long missingMaxPossible;
|
||||
// the max possible count for a missing term for each shard (indexed by shardNum)
|
||||
long[] missingMax;
|
||||
OpenBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
|
||||
HashMap<String,ShardFacetCount> counts = new HashMap<String,ShardFacetCount>(128);
|
||||
int termNum;
|
||||
String queryPrefix;
|
||||
|
||||
int initialLimit; // how many terms requested in first phase
|
||||
boolean needRefinements;
|
||||
ShardFacetCount[] countSorted;
|
||||
|
||||
DistribFieldFacet(ResponseBuilder rb, String field) {
|
||||
sf = rb.req.getSchema().getField(field);
|
||||
missingMax = new long[rb.shards.length];
|
||||
counted = new OpenBitSet[rb.shards.length];
|
||||
queryPrefix = "<!field f=" + field + '>';
|
||||
}
|
||||
|
||||
void add(int shardNum, NamedList shardCounts, int numRequested) {
|
||||
int sz = shardCounts.size();
|
||||
int numReceived = sz;
|
||||
|
||||
OpenBitSet terms = new OpenBitSet(termNum+sz);
|
||||
|
||||
long last = 0;
|
||||
for (int i=0; i<sz; i++) {
|
||||
String name = shardCounts.getName(i);
|
||||
long count = ((Number)shardCounts.getVal(i)).longValue();
|
||||
if (name == null) {
|
||||
missingCount += count;
|
||||
numReceived--;
|
||||
} else {
|
||||
ShardFacetCount sfc = counts.get(name);
|
||||
if (sfc == null) {
|
||||
sfc = new ShardFacetCount();
|
||||
sfc.name = name;
|
||||
sfc.termNum = termNum++;
|
||||
counts.put(name, sfc);
|
||||
}
|
||||
sfc.count += count;
|
||||
terms.fastSet(sfc.termNum);
|
||||
last = count;
|
||||
}
|
||||
}
|
||||
|
||||
// the largest possible missing term is 0 if we received less
|
||||
// than the number requested (provided mincount==0 like it should be for
|
||||
// a shard request)
|
||||
if (numRequested !=0 && numReceived < numRequested) {
|
||||
last = 0;
|
||||
}
|
||||
|
||||
missingMaxPossible += last;
|
||||
missingMax[shardNum] = last;
|
||||
counted[shardNum] = terms;
|
||||
}
|
||||
|
||||
|
||||
ShardFacetCount[] getSorted() {
|
||||
ShardFacetCount[] arr = counts.values().toArray(new ShardFacetCount[counts.size()]);
|
||||
Arrays.sort(arr, new Comparator<ShardFacetCount>() {
|
||||
public int compare(ShardFacetCount o1, ShardFacetCount o2) {
|
||||
if (o2.count < o1.count) return -1;
|
||||
else if (o1.count < o2.count) return 1;
|
||||
// TODO: handle tiebreaks for types other than strings
|
||||
return o1.name.compareTo(o2.name);
|
||||
}
|
||||
});
|
||||
countSorted = arr;
|
||||
return arr;
|
||||
}
|
||||
|
||||
String makeQuery(ShardFacetCount sfc) {
|
||||
return queryPrefix + sfc.name;
|
||||
}
|
||||
|
||||
// returns the max possible value this ShardFacetCount could have for this shard
|
||||
// (assumes the shard did not report a count for this value)
|
||||
long maxPossible(ShardFacetCount sfc, int shardNum) {
|
||||
return missingMax[shardNum];
|
||||
// TODO: could store the last term in the shard to tell if this term
|
||||
// comes before or after it. If it comes before, we could subtract 1
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
class ShardFacetCount {
|
||||
String name;
|
||||
long count;
|
||||
int termNum; // term number starting at 0 (used in bit arrays)
|
||||
|
||||
public String toString() {
|
||||
return "{term="+name+",termNum="+termNum+",count="+count+"}";
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,14 +19,16 @@ package org.apache.solr.handler.component;
|
|||
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.HighlightParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.highlight.SolrHighlighter;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryResponse;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* TODO!
|
||||
|
@ -39,51 +41,113 @@ public class HighlightComponent extends SearchComponent
|
|||
public static final String COMPONENT_NAME = "highlight";
|
||||
|
||||
@Override
|
||||
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp)
|
||||
public void prepare(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
|
||||
SolrHighlighter highlighter = rb.req.getCore().getHighlighter();
|
||||
rb.doHighlights = highlighter.isHighlightingEnabled(rb.req.getParams());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
|
||||
SolrHighlighter highlighter = req.getCore().getHighlighter();
|
||||
if (highlighter.isHighlightingEnabled(req.getParams())) {
|
||||
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
|
||||
public void process(ResponseBuilder rb) throws IOException {
|
||||
SolrQueryRequest req = rb.req;
|
||||
if (rb.doHighlights) {
|
||||
SolrHighlighter highlighter = req.getCore().getHighlighter();
|
||||
SolrParams params = req.getParams();
|
||||
|
||||
String[] defaultHighlightFields; //TODO: get from builder by default?
|
||||
|
||||
if (builder.getQparser() != null) {
|
||||
defaultHighlightFields = builder.getQparser().getDefaultHighlightFields();
|
||||
if (rb.getQparser() != null) {
|
||||
defaultHighlightFields = rb.getQparser().getDefaultHighlightFields();
|
||||
} else {
|
||||
defaultHighlightFields = params.getParams(CommonParams.DF);
|
||||
}
|
||||
|
||||
if(builder.getHighlightQuery()==null) {
|
||||
if (builder.getQparser() != null) {
|
||||
if(rb.getHighlightQuery()==null) {
|
||||
if (rb.getQparser() != null) {
|
||||
try {
|
||||
builder.setHighlightQuery( builder.getQparser().getHighlightQuery() );
|
||||
rb.setHighlightQuery( rb.getQparser().getHighlightQuery() );
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
|
||||
}
|
||||
} else {
|
||||
builder.setHighlightQuery( builder.getQuery() );
|
||||
rb.setHighlightQuery( rb.getQuery() );
|
||||
}
|
||||
}
|
||||
|
||||
NamedList sumData = highlighter.doHighlighting(
|
||||
builder.getResults().docList,
|
||||
builder.getHighlightQuery().rewrite(req.getSearcher().getReader()),
|
||||
rb.getResults().docList,
|
||||
rb.getHighlightQuery().rewrite(req.getSearcher().getReader()),
|
||||
req, defaultHighlightFields );
|
||||
|
||||
if(sumData != null) {
|
||||
// TODO ???? add this directly to the response?
|
||||
rsp.add("highlighting", sumData);
|
||||
rb.rsp.add("highlighting", sumData);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////
|
||||
|
||||
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
|
||||
if (!rb.doHighlights) return;
|
||||
|
||||
// Turn on highlighting only only when retrieving fields
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
|
||||
sreq.purpose |= ShardRequest.PURPOSE_GET_HIGHLIGHTS;
|
||||
// should already be true...
|
||||
sreq.params.set(HighlightParams.HIGHLIGHT, "true");
|
||||
} else {
|
||||
sreq.params.set(HighlightParams.HIGHLIGHT, "false");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finishStage(ResponseBuilder rb) {
|
||||
if (rb.doHighlights && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
|
||||
NamedList hlResult = new SimpleOrderedMap();
|
||||
|
||||
Object[] arr = new Object[rb.resultIds.size() * 2];
|
||||
|
||||
// TODO: make a generic routine to do automatic merging of id keyed data
|
||||
for (ShardRequest sreq : rb.finished) {
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_HIGHLIGHTS) == 0) continue;
|
||||
for (ShardResponse srsp : sreq.responses) {
|
||||
NamedList hl = (NamedList)srsp.rsp.getResponse().get("highlighting");
|
||||
for (int i=0; i<hl.size(); i++) {
|
||||
String id = hl.getName(i);
|
||||
ShardDoc sdoc = rb.resultIds.get(id);
|
||||
int idx = sdoc.positionInResponse;
|
||||
arr[idx<<1] = id;
|
||||
arr[(idx<<1)+1] = hl.getVal(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// remove nulls in case not all docs were able to be retrieved
|
||||
rb.rsp.add("highlighting", removeNulls(new NamedList(Arrays.asList(arr))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static NamedList removeNulls(NamedList nl) {
|
||||
for (int i=0; i<nl.size(); i++) {
|
||||
if (nl.getName(i)==null) {
|
||||
NamedList newList = new NamedList();
|
||||
for (int j=0; j<nl.size(); j++) {
|
||||
String n = nl.getName(j);
|
||||
if (n != null) {
|
||||
newList.add(n, nl.getVal(j));
|
||||
}
|
||||
}
|
||||
return newList;
|
||||
}
|
||||
}
|
||||
return nl;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////
|
||||
/// SolrInfoMBean
|
||||
////////////////////////////////////////////
|
||||
|
||||
|
|
|
@ -24,8 +24,6 @@ import org.apache.solr.common.params.MoreLikeThisParams;
|
|||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.handler.MoreLikeThisHandler;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryResponse;
|
||||
import org.apache.solr.search.DocList;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
|
||||
|
@ -40,28 +38,27 @@ public class MoreLikeThisComponent extends SearchComponent
|
|||
public static final String COMPONENT_NAME = "mlt";
|
||||
|
||||
@Override
|
||||
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
|
||||
public void prepare(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
|
||||
public void process(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
SolrParams p = req.getParams();
|
||||
SolrParams p = rb.req.getParams();
|
||||
if( p.getBool( MoreLikeThisParams.MLT, false ) ) {
|
||||
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
|
||||
SolrIndexSearcher searcher = req.getSearcher();
|
||||
SolrIndexSearcher searcher = rb.req.getSearcher();
|
||||
|
||||
MoreLikeThisHandler.MoreLikeThisHelper mlt
|
||||
= new MoreLikeThisHandler.MoreLikeThisHelper( p, searcher );
|
||||
|
||||
int mltcount = p.getInt( MoreLikeThisParams.DOC_COUNT, 5 );
|
||||
NamedList<DocList> sim = mlt.getMoreLikeThese(
|
||||
builder.getResults().docList, mltcount, builder.getFieldFlags() );
|
||||
rb.getResults().docList, mltcount, rb.getFieldFlags() );
|
||||
|
||||
// TODO ???? add this directly to the response?
|
||||
rsp.add( "moreLikeThis", sim );
|
||||
rb.rsp.add( "moreLikeThis", sim );
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -17,19 +17,32 @@
|
|||
|
||||
package org.apache.solr.handler.component;
|
||||
|
||||
import org.apache.lucene.document.Fieldable;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.queryParser.ParseException;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryResponse;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.search.*;
|
||||
import org.apache.solr.util.SolrPluginUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.*;
|
||||
import java.text.Collator;
|
||||
|
||||
/**
|
||||
* TODO!
|
||||
|
@ -42,9 +55,10 @@ public class QueryComponent extends SearchComponent
|
|||
public static final String COMPONENT_NAME = "query";
|
||||
|
||||
@Override
|
||||
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, ParseException
|
||||
public void prepare(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
|
||||
SolrQueryRequest req = rb.req;
|
||||
SolrQueryResponse rsp = rb.rsp;
|
||||
SolrParams params = req.getParams();
|
||||
|
||||
// Set field flags
|
||||
|
@ -53,32 +67,43 @@ public class QueryComponent extends SearchComponent
|
|||
if (fl != null) {
|
||||
fieldFlags |= SolrPluginUtils.setReturnFields(fl, rsp);
|
||||
}
|
||||
builder.setFieldFlags( fieldFlags );
|
||||
rb.setFieldFlags( fieldFlags );
|
||||
|
||||
String defType = params.get(QueryParsing.DEFTYPE);
|
||||
defType = defType==null ? OldLuceneQParserPlugin.NAME : defType;
|
||||
|
||||
if (builder.getQueryString() == null) {
|
||||
builder.setQueryString( params.get( CommonParams.Q ) );
|
||||
if (rb.getQueryString() == null) {
|
||||
rb.setQueryString( params.get( CommonParams.Q ) );
|
||||
}
|
||||
|
||||
QParser parser = QParser.getParser(builder.getQueryString(), defType, req);
|
||||
builder.setQuery( parser.getQuery() );
|
||||
builder.setSortSpec( parser.getSort(true) );
|
||||
|
||||
String[] fqs = req.getParams().getParams(org.apache.solr.common.params.CommonParams.FQ);
|
||||
if (fqs!=null && fqs.length!=0) {
|
||||
List<Query> filters = builder.getFilters();
|
||||
if (filters==null) {
|
||||
filters = new ArrayList<Query>();
|
||||
builder.setFilters( filters );
|
||||
}
|
||||
for (String fq : fqs) {
|
||||
if (fq != null && fq.trim().length()!=0) {
|
||||
QParser fqp = QParser.getParser(fq, null, req);
|
||||
filters.add(fqp.getQuery());
|
||||
try {
|
||||
QParser parser = QParser.getParser(rb.getQueryString(), defType, req);
|
||||
rb.setQuery( parser.getQuery() );
|
||||
rb.setSortSpec( parser.getSort(true) );
|
||||
|
||||
String[] fqs = req.getParams().getParams(org.apache.solr.common.params.CommonParams.FQ);
|
||||
if (fqs!=null && fqs.length!=0) {
|
||||
List<Query> filters = rb.getFilters();
|
||||
if (filters==null) {
|
||||
filters = new ArrayList<Query>();
|
||||
rb.setFilters( filters );
|
||||
}
|
||||
for (String fq : fqs) {
|
||||
if (fq != null && fq.trim().length()!=0) {
|
||||
QParser fqp = QParser.getParser(fq, null, req);
|
||||
filters.add(fqp.getQuery());
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (ParseException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
|
||||
}
|
||||
|
||||
// TODO: temporary... this should go in a different component.
|
||||
String shards = params.get("shards");
|
||||
if (shards != null) {
|
||||
List<String> lst = StrUtils.splitSmart(shards, ",", true);
|
||||
rb.shards = lst.toArray(new String[lst.size()]);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -86,35 +111,692 @@ public class QueryComponent extends SearchComponent
|
|||
* Actually run the query
|
||||
*/
|
||||
@Override
|
||||
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
|
||||
public void process(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
|
||||
SolrQueryRequest req = rb.req;
|
||||
SolrQueryResponse rsp = rb.rsp;
|
||||
SolrIndexSearcher searcher = req.getSearcher();
|
||||
SolrParams params = req.getParams();
|
||||
|
||||
if( builder.isNeedDocSet() ) {
|
||||
builder.setResults( searcher.getDocListAndSet(
|
||||
builder.getQuery(), builder.getFilters(), builder.getSortSpec().getSort(),
|
||||
builder.getSortSpec().getOffset(), builder.getSortSpec().getCount(),
|
||||
builder.getFieldFlags() ) );
|
||||
|
||||
// Optional: This could also be implemented by the top-level searcher sending
|
||||
// a filter that lists the ids... that would be transparent to
|
||||
// the request handler, but would be more expensive (and would preserve score
|
||||
// too if desired).
|
||||
String ids = params.get("ids");
|
||||
if (ids != null) {
|
||||
SchemaField idField = req.getSchema().getUniqueKeyField();
|
||||
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
|
||||
int[] luceneIds = new int[idArr.size()];
|
||||
int docs = 0;
|
||||
for (int i=0; i<idArr.size(); i++) {
|
||||
int id = req.getSearcher().getFirstMatch(
|
||||
new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
|
||||
if (id >= 0)
|
||||
luceneIds[docs++] = id;
|
||||
}
|
||||
|
||||
DocListAndSet res = new DocListAndSet();
|
||||
res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
|
||||
if (rb.isNeedDocSet()) {
|
||||
List<Query> queries = new ArrayList<Query>();
|
||||
queries.add(rb.getQuery());
|
||||
List<Query> filters = rb.getFilters();
|
||||
if (filters != null) queries.addAll(filters);
|
||||
res.docSet = searcher.getDocSet(queries);
|
||||
}
|
||||
rb.setResults(res);
|
||||
rsp.add("response",rb.getResults().docList);
|
||||
return;
|
||||
}
|
||||
|
||||
if( rb.isNeedDocSet() ) {
|
||||
rb.setResults( searcher.getDocListAndSet(
|
||||
rb.getQuery(), rb.getFilters(), rb.getSortSpec().getSort(),
|
||||
rb.getSortSpec().getOffset(), rb.getSortSpec().getCount(),
|
||||
rb.getFieldFlags() ) );
|
||||
}
|
||||
else {
|
||||
DocListAndSet results = new DocListAndSet();
|
||||
results.docList = searcher.getDocList(
|
||||
builder.getQuery(), builder.getFilters(), builder.getSortSpec().getSort(),
|
||||
builder.getSortSpec().getOffset(), builder.getSortSpec().getCount(),
|
||||
builder.getFieldFlags() );
|
||||
builder.setResults( results );
|
||||
rb.getQuery(), rb.getFilters(), rb.getSortSpec().getSort(),
|
||||
rb.getSortSpec().getOffset(), rb.getSortSpec().getCount(),
|
||||
rb.getFieldFlags() );
|
||||
rb.setResults( results );
|
||||
}
|
||||
|
||||
rsp.add("response",rb.getResults().docList);
|
||||
|
||||
|
||||
boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES,false);
|
||||
if(fsv){
|
||||
Sort sort = rb.getSortSpec().getSort();
|
||||
SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort();
|
||||
ScoreDoc sd = new ScoreDoc(0,1.0f); // won't work for comparators that look at the score
|
||||
NamedList sortVals = new NamedList();
|
||||
StringFieldable field = new StringFieldable();
|
||||
|
||||
for (SortField sortField: sortFields) {
|
||||
int type = sortField.getType();
|
||||
if (type==SortField.SCORE || type==SortField.DOC) continue;
|
||||
|
||||
ScoreDocComparator comparator = null;
|
||||
IndexReader reader = searcher.getReader();
|
||||
String fieldname = sortField.getField();
|
||||
FieldType ft = fieldname==null ? null : req.getSchema().getFieldTypeNoEx(fieldname);
|
||||
|
||||
|
||||
switch (type) {
|
||||
case SortField.INT:
|
||||
comparator = comparatorInt (reader, fieldname);
|
||||
break;
|
||||
case SortField.FLOAT:
|
||||
comparator = comparatorFloat (reader, fieldname);
|
||||
break;
|
||||
case SortField.LONG:
|
||||
comparator = comparatorLong(reader, fieldname);
|
||||
break;
|
||||
case SortField.DOUBLE:
|
||||
comparator = comparatorDouble(reader, fieldname);
|
||||
break;
|
||||
case SortField.STRING:
|
||||
if (sortField.getLocale() != null) comparator = comparatorStringLocale (reader, fieldname, sortField.getLocale());
|
||||
else comparator = comparatorString (reader, fieldname);
|
||||
break;
|
||||
case SortField.CUSTOM:
|
||||
comparator = sortField.getFactory().newComparator (reader, fieldname);
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException ("unknown field type: "+type);
|
||||
}
|
||||
|
||||
DocList docList = rb.getResults().docList;
|
||||
ArrayList<Object> vals = new ArrayList<Object>(docList.size());
|
||||
DocIterator it = rb.getResults().docList.iterator();
|
||||
while(it.hasNext()) {
|
||||
sd.doc = it.nextDoc();
|
||||
Object val = comparator.sortValue(sd);
|
||||
// Sortable float, double, int, long types all just use a string
|
||||
// comparator. For these, we need to put the type into a readable
|
||||
// format. One reason for this is that XML can't represent all
|
||||
// string values (or even all unicode code points).
|
||||
// indexedToReadable() should be a no-op and should
|
||||
// thus be harmless anyway (for all current ways anyway)
|
||||
if (val instanceof String) {
|
||||
field.val = (String)val;
|
||||
val = ft.toObject(field);
|
||||
}
|
||||
vals.add(val);
|
||||
}
|
||||
|
||||
sortVals.add(fieldname, vals);
|
||||
}
|
||||
|
||||
rsp.add("sort_values", sortVals);
|
||||
}
|
||||
|
||||
//pre-fetch returned documents
|
||||
if (builder.getResults().docList != null && builder.getResults().docList.size()<=50) {
|
||||
if (!req.getParams().getBool("isShard",false) && rb.getResults().docList != null && rb.getResults().docList.size()<=50) {
|
||||
// TODO: this may depend on the highlighter component (or other components?)
|
||||
SolrPluginUtils.optimizePreFetchDocs(builder.getResults().docList, builder.getQuery(), req, rsp);
|
||||
SolrPluginUtils.optimizePreFetchDocs(rb.getResults().docList, rb.getQuery(), req, rsp);
|
||||
}
|
||||
rsp.add("response",builder.getResults().docList);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int distributedProcess(ResponseBuilder rb) throws IOException {
|
||||
if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY)
|
||||
return ResponseBuilder.STAGE_PARSE_QUERY;
|
||||
if (rb.stage == ResponseBuilder.STAGE_PARSE_QUERY) {
|
||||
createDistributedIdf(rb);
|
||||
return ResponseBuilder.STAGE_EXECUTE_QUERY;
|
||||
}
|
||||
if (rb.stage < ResponseBuilder.STAGE_EXECUTE_QUERY) return ResponseBuilder.STAGE_EXECUTE_QUERY;
|
||||
if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) {
|
||||
createMainQuery(rb);
|
||||
return ResponseBuilder.STAGE_GET_FIELDS;
|
||||
}
|
||||
if (rb.stage < ResponseBuilder.STAGE_GET_FIELDS) return ResponseBuilder.STAGE_GET_FIELDS;
|
||||
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
|
||||
createRetrieveDocs(rb);
|
||||
return ResponseBuilder.STAGE_DONE;
|
||||
}
|
||||
return ResponseBuilder.STAGE_DONE;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
|
||||
mergeIds(rb, sreq);
|
||||
return;
|
||||
}
|
||||
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
|
||||
returnFields(rb, sreq);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finishStage(ResponseBuilder rb) {
|
||||
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
|
||||
// We may not have been able to retrieve all the docs due to an
|
||||
// index change. Remove any null documents.
|
||||
for (Iterator<SolrDocument> iter = rb._responseDocs.iterator(); iter.hasNext();) {
|
||||
if (iter.next() == null) {
|
||||
iter.remove();
|
||||
rb._responseDocs.setNumFound(rb._responseDocs.getNumFound()-1);
|
||||
}
|
||||
}
|
||||
|
||||
rb.rsp.add("response", rb._responseDocs);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void createDistributedIdf(ResponseBuilder rb) {
|
||||
// TODO
|
||||
}
|
||||
|
||||
private void createMainQuery(ResponseBuilder rb) {
|
||||
ShardRequest sreq = new ShardRequest();
|
||||
sreq.purpose = ShardRequest.PURPOSE_GET_TOP_IDS;
|
||||
|
||||
sreq.params = new ModifiableSolrParams(rb.req.getParams());
|
||||
// TODO: base on current params or original params?
|
||||
|
||||
// don't pass through any shards param
|
||||
sreq.params.remove("shards");
|
||||
|
||||
// set the start (offset) to 0 for each shard request so we can properly merge
|
||||
// results from the start.
|
||||
sreq.params.set("start","0");
|
||||
|
||||
// TODO: should we even use the SortSpec? That's obtained from the QParser, and
|
||||
// perhaps we shouldn't attempt to parse the query at this level?
|
||||
// Alternate Idea: instead of specifying all these things at the upper level,
|
||||
// we could just specify that this is a shard request.
|
||||
sreq.params.set("rows", rb.getSortSpec().getOffset() + rb.getSortSpec().getCount());
|
||||
|
||||
|
||||
// in this first phase, request only the unique key field
|
||||
// and any fields needed for merging.
|
||||
sreq.params.set(ResponseBuilder.FIELD_SORT_VALUES,"true");
|
||||
|
||||
if (rb.getSortSpec().includesScore()) {
|
||||
sreq.params.set("fl", rb.req.getSchema().getUniqueKeyField().getName() + ",score");
|
||||
} else {
|
||||
sreq.params.set("fl", rb.req.getSchema().getUniqueKeyField().getName());
|
||||
}
|
||||
|
||||
rb.addRequest(this, sreq);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private void mergeIds(ResponseBuilder rb, ShardRequest sreq) {
|
||||
SortSpec ss = rb.getSortSpec();
|
||||
Sort sort = ss.getSort();
|
||||
|
||||
SortField[] sortFields = null;
|
||||
if(sort != null) sortFields = sort.getSort();
|
||||
else {
|
||||
sortFields = new SortField[]{SortField.FIELD_SCORE};
|
||||
}
|
||||
|
||||
SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField();
|
||||
|
||||
|
||||
// id to shard mapping, to eliminate any accidental dups
|
||||
HashMap<Object,String> uniqueDoc = new HashMap<Object,String>();
|
||||
|
||||
// Merge the docs via a priority queue so we don't have to sort *all* of the
|
||||
// documents... we only need to order the top (rows+start)
|
||||
ShardFieldSortedHitQueue queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount());
|
||||
|
||||
long numFound = 0;
|
||||
Float maxScore=null;
|
||||
for (ShardResponse srsp : sreq.responses) {
|
||||
SolrDocumentList docs = (SolrDocumentList)srsp.rsp.getResponse().get("response");
|
||||
|
||||
// calculate global maxScore and numDocsFound
|
||||
if (docs.getMaxScore() != null) {
|
||||
maxScore = maxScore==null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore());
|
||||
}
|
||||
numFound += docs.getNumFound();
|
||||
|
||||
NamedList sortFieldValues = (NamedList)(srsp.rsp.getResponse().get("sort_values"));
|
||||
|
||||
// go through every doc in this response, construct a ShardDoc, and
|
||||
// put it in the priority queue so it can be ordered.
|
||||
for (int i=0; i<docs.size(); i++) {
|
||||
SolrDocument doc = docs.get(i);
|
||||
Object id = doc.getFieldValue(uniqueKeyField.getName());
|
||||
|
||||
String prevShard = uniqueDoc.put(id, srsp.shard);
|
||||
if (prevShard != null) {
|
||||
// duplicate detected
|
||||
numFound--;
|
||||
|
||||
// For now, just always use the first encountered since we can't currently
|
||||
// remove the previous one added to the priority queue. If we switched
|
||||
// to the Java5 PriorityQueue, this would be easier.
|
||||
continue;
|
||||
// make which duplicate is used deterministic based on shard
|
||||
// if (prevShard.compareTo(srsp.shard) >= 0) {
|
||||
// TODO: remove previous from priority queue
|
||||
// continue;
|
||||
// }
|
||||
}
|
||||
|
||||
ShardDoc shardDoc = new ShardDoc();
|
||||
shardDoc.id = id;
|
||||
shardDoc.shard = srsp.shard;
|
||||
shardDoc.orderInShard = i;
|
||||
Object scoreObj = doc.getFieldValue("score");
|
||||
if (scoreObj != null) {
|
||||
if (scoreObj instanceof String) {
|
||||
shardDoc.score = Float.parseFloat((String)scoreObj);
|
||||
} else {
|
||||
shardDoc.score = (Float)scoreObj;
|
||||
}
|
||||
}
|
||||
|
||||
shardDoc.sortFieldValues = sortFieldValues;
|
||||
|
||||
queue.insert(shardDoc);
|
||||
} // end for-each-doc-in-response
|
||||
} // end for-each-response
|
||||
|
||||
|
||||
// The queue now has 0 -> queuesize docs, where queuesize <= start + rows
|
||||
// So we want to pop the last documents off the queue to get
|
||||
// the docs offset -> queuesize
|
||||
int resultSize = queue.size() - ss.getOffset();
|
||||
resultSize = Math.max(0, resultSize); // there may not be any docs in range
|
||||
|
||||
Map<Object,ShardDoc> resultIds = new HashMap<Object,ShardDoc>();
|
||||
for (int i=resultSize-1; i>=0; i--) {
|
||||
ShardDoc shardDoc = (ShardDoc)queue.pop();
|
||||
shardDoc.positionInResponse = i;
|
||||
// Need the toString() for correlation with other lists that must
|
||||
// be strings (like keys in highlighting, explain, etc)
|
||||
resultIds.put(shardDoc.id.toString(), shardDoc);
|
||||
}
|
||||
|
||||
|
||||
SolrDocumentList responseDocs = new SolrDocumentList();
|
||||
if (maxScore!=null) responseDocs.setMaxScore(maxScore);
|
||||
responseDocs.setNumFound(numFound);
|
||||
responseDocs.setStart(ss.getOffset());
|
||||
|
||||
// size appropriately
|
||||
for (int i=0; i<resultSize; i++) responseDocs.add(null);
|
||||
|
||||
// save these results in a private area so we can access them
|
||||
// again when retrieving stored fields.
|
||||
// TODO: use ResponseBuilder (w/ comments) or the request context?
|
||||
rb.resultIds = resultIds;
|
||||
rb._responseDocs = responseDocs;
|
||||
}
|
||||
|
||||
private void createRetrieveDocs(ResponseBuilder rb) {
|
||||
|
||||
// TODO: in a system with nTiers > 2, we could be passed "ids" here
|
||||
// unless those requests always go to the final destination shard
|
||||
|
||||
// for each shard, collect the documents for that shard.
|
||||
HashMap<String, Collection<ShardDoc>> shardMap = new HashMap<String,Collection<ShardDoc>>();
|
||||
for (ShardDoc sdoc : rb.resultIds.values()) {
|
||||
Collection<ShardDoc> shardDocs = shardMap.get(sdoc.shard);
|
||||
if (shardDocs == null) {
|
||||
shardDocs = new ArrayList<ShardDoc>();
|
||||
shardMap.put(sdoc.shard, shardDocs);
|
||||
}
|
||||
shardDocs.add(sdoc);
|
||||
}
|
||||
|
||||
SchemaField uniqueField = rb.req.getSchema().getUniqueKeyField();
|
||||
|
||||
// Now create a request for each shard to retrieve the stored fields
|
||||
for (Collection<ShardDoc> shardDocs : shardMap.values()) {
|
||||
ShardRequest sreq = new ShardRequest();
|
||||
sreq.purpose = ShardRequest.PURPOSE_GET_FIELDS;
|
||||
|
||||
sreq.shards = new String[] {shardDocs.iterator().next().shard};
|
||||
|
||||
sreq.params = new ModifiableSolrParams();
|
||||
|
||||
// add original params
|
||||
sreq.params.add( rb.req.getParams());
|
||||
|
||||
// no need for a sort, we already have order
|
||||
sreq.params.remove("sort");
|
||||
|
||||
// we already have the field sort values
|
||||
sreq.params.remove(ResponseBuilder.FIELD_SORT_VALUES);
|
||||
|
||||
// make sure that the id is returned for correlation
|
||||
String fl = sreq.params.get("fl");
|
||||
if (fl != null) {
|
||||
sreq.params.set("fl", fl+','+uniqueField.getName());
|
||||
}
|
||||
|
||||
ArrayList<String> ids = new ArrayList<String>(shardDocs.size());
|
||||
for (ShardDoc shardDoc : shardDocs) {
|
||||
// TODO: depending on the type, we may need more tha a simple toString()?
|
||||
ids.add(shardDoc.id.toString());
|
||||
}
|
||||
sreq.params.add("ids", StrUtils.join(ids, ','));
|
||||
|
||||
rb.addRequest(this, sreq);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void returnFields(ResponseBuilder rb, ShardRequest sreq) {
|
||||
// Keep in mind that this could also be a shard in a multi-tiered system.
|
||||
// TODO: if a multi-tiered system, it seems like some requests
|
||||
// could/should bypass middlemen (like retrieving stored fields)
|
||||
// TODO: merge fsv to if requested
|
||||
|
||||
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
|
||||
boolean returnScores = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0;
|
||||
|
||||
assert(sreq.responses.size() == 1);
|
||||
ShardResponse srsp = sreq.responses.get(0);
|
||||
SolrDocumentList docs = (SolrDocumentList)srsp.rsp.getResponse().get("response");
|
||||
|
||||
String keyFieldName = rb.req.getSchema().getUniqueKeyField().getName();
|
||||
|
||||
for (SolrDocument doc : docs) {
|
||||
Object id = doc.getFieldValue(keyFieldName);
|
||||
ShardDoc sdoc = rb.resultIds.get(id.toString());
|
||||
if (returnScores && sdoc.score != null) {
|
||||
doc.setField("score", sdoc.score);
|
||||
}
|
||||
rb._responseDocs.set(sdoc.positionInResponse, doc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/////////////////////////////////////////////
|
||||
/// Comparators copied from Lucene
|
||||
/////////////////////////////////////////////
|
||||
|
||||
/**
|
||||
* Returns a comparator for sorting hits according to a field containing integers.
|
||||
* @param reader Index to use.
|
||||
* @param fieldname Fieldable containg integer values.
|
||||
* @return Comparator for sorting hits.
|
||||
* @throws IOException If an error occurs reading the index.
|
||||
*/
|
||||
static ScoreDocComparator comparatorInt (final IndexReader reader, final String fieldname)
|
||||
throws IOException {
|
||||
final String field = fieldname.intern();
|
||||
final int[] fieldOrder = FieldCache.DEFAULT.getInts (reader, field);
|
||||
return new ScoreDocComparator() {
|
||||
|
||||
public final int compare (final ScoreDoc i, final ScoreDoc j) {
|
||||
final int fi = fieldOrder[i.doc];
|
||||
final int fj = fieldOrder[j.doc];
|
||||
if (fi < fj) return -1;
|
||||
if (fi > fj) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
public Comparable sortValue (final ScoreDoc i) {
|
||||
return new Integer (fieldOrder[i.doc]);
|
||||
}
|
||||
|
||||
public int sortType() {
|
||||
return SortField.INT;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a comparator for sorting hits according to a field containing integers.
|
||||
* @param reader Index to use.
|
||||
* @param fieldname Fieldable containg integer values.
|
||||
* @return Comparator for sorting hits.
|
||||
* @throws IOException If an error occurs reading the index.
|
||||
*/
|
||||
static ScoreDocComparator comparatorLong (final IndexReader reader, final String fieldname)
|
||||
throws IOException {
|
||||
final String field = fieldname.intern();
|
||||
final long[] fieldOrder = ExtendedFieldCache.EXT_DEFAULT.getLongs (reader, field);
|
||||
return new ScoreDocComparator() {
|
||||
|
||||
public final int compare (final ScoreDoc i, final ScoreDoc j) {
|
||||
final long li = fieldOrder[i.doc];
|
||||
final long lj = fieldOrder[j.doc];
|
||||
if (li < lj) return -1;
|
||||
if (li > lj) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
public Comparable sortValue (final ScoreDoc i) {
|
||||
return new Long(fieldOrder[i.doc]);
|
||||
}
|
||||
|
||||
public int sortType() {
|
||||
return SortField.LONG;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a comparator for sorting hits according to a field containing floats.
|
||||
* @param reader Index to use.
|
||||
* @param fieldname Fieldable containg float values.
|
||||
* @return Comparator for sorting hits.
|
||||
* @throws IOException If an error occurs reading the index.
|
||||
*/
|
||||
static ScoreDocComparator comparatorFloat (final IndexReader reader, final String fieldname)
|
||||
throws IOException {
|
||||
final String field = fieldname.intern();
|
||||
final float[] fieldOrder = FieldCache.DEFAULT.getFloats (reader, field);
|
||||
return new ScoreDocComparator () {
|
||||
|
||||
public final int compare (final ScoreDoc i, final ScoreDoc j) {
|
||||
final float fi = fieldOrder[i.doc];
|
||||
final float fj = fieldOrder[j.doc];
|
||||
if (fi < fj) return -1;
|
||||
if (fi > fj) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
public Comparable sortValue (final ScoreDoc i) {
|
||||
return new Float (fieldOrder[i.doc]);
|
||||
}
|
||||
|
||||
public int sortType() {
|
||||
return SortField.FLOAT;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns a comparator for sorting hits according to a field containing doubles.
|
||||
* @param reader Index to use.
|
||||
* @param fieldname Fieldable containg float values.
|
||||
* @return Comparator for sorting hits.
|
||||
* @throws IOException If an error occurs reading the index.
|
||||
*/
|
||||
static ScoreDocComparator comparatorDouble(final IndexReader reader, final String fieldname)
|
||||
throws IOException {
|
||||
final String field = fieldname.intern();
|
||||
final double[] fieldOrder = ExtendedFieldCache.EXT_DEFAULT.getDoubles (reader, field);
|
||||
return new ScoreDocComparator () {
|
||||
|
||||
public final int compare (final ScoreDoc i, final ScoreDoc j) {
|
||||
final double di = fieldOrder[i.doc];
|
||||
final double dj = fieldOrder[j.doc];
|
||||
if (di < dj) return -1;
|
||||
if (di > dj) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
public Comparable sortValue (final ScoreDoc i) {
|
||||
return new Double (fieldOrder[i.doc]);
|
||||
}
|
||||
|
||||
public int sortType() {
|
||||
return SortField.DOUBLE;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a comparator for sorting hits according to a field containing strings.
|
||||
* @param reader Index to use.
|
||||
* @param fieldname Fieldable containg string values.
|
||||
* @return Comparator for sorting hits.
|
||||
* @throws IOException If an error occurs reading the index.
|
||||
*/
|
||||
static ScoreDocComparator comparatorString (final IndexReader reader, final String fieldname)
|
||||
throws IOException {
|
||||
final String field = fieldname.intern();
|
||||
final FieldCache.StringIndex index = FieldCache.DEFAULT.getStringIndex (reader, field);
|
||||
return new ScoreDocComparator () {
|
||||
|
||||
public final int compare (final ScoreDoc i, final ScoreDoc j) {
|
||||
final int fi = index.order[i.doc];
|
||||
final int fj = index.order[j.doc];
|
||||
if (fi < fj) return -1;
|
||||
if (fi > fj) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
public Comparable sortValue (final ScoreDoc i) {
|
||||
return index.lookup[index.order[i.doc]];
|
||||
}
|
||||
|
||||
public int sortType() {
|
||||
return SortField.STRING;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a comparator for sorting hits according to a field containing strings.
|
||||
* @param reader Index to use.
|
||||
* @param fieldname Fieldable containg string values.
|
||||
* @return Comparator for sorting hits.
|
||||
* @throws IOException If an error occurs reading the index.
|
||||
*/
|
||||
static ScoreDocComparator comparatorStringLocale (final IndexReader reader, final String fieldname, final Locale locale)
|
||||
throws IOException {
|
||||
final Collator collator = Collator.getInstance (locale);
|
||||
final String field = fieldname.intern();
|
||||
final String[] index = FieldCache.DEFAULT.getStrings (reader, field);
|
||||
return new ScoreDocComparator() {
|
||||
|
||||
public final int compare(final ScoreDoc i, final ScoreDoc j) {
|
||||
String is = index[i.doc];
|
||||
String js = index[j.doc];
|
||||
if (is == js) {
|
||||
return 0;
|
||||
} else if (is == null) {
|
||||
return -1;
|
||||
} else if (js == null) {
|
||||
return 1;
|
||||
} else {
|
||||
return collator.compare(is, js);
|
||||
}
|
||||
}
|
||||
|
||||
public Comparable sortValue (final ScoreDoc i) {
|
||||
return index[i.doc];
|
||||
}
|
||||
|
||||
public int sortType() {
|
||||
return SortField.STRING;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
static class StringFieldable implements Fieldable {
|
||||
public String val;
|
||||
|
||||
public void setBoost(float boost) {
|
||||
}
|
||||
|
||||
public float getBoost() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
public String name() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public String stringValue() {
|
||||
return val;
|
||||
}
|
||||
|
||||
public Reader readerValue() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public byte[] binaryValue() {
|
||||
return new byte[0];
|
||||
}
|
||||
|
||||
public TokenStream tokenStreamValue() {
|
||||
return null;
|
||||
}
|
||||
|
||||
public boolean isStored() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean isIndexed() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean isTokenized() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean isCompressed() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isTermVectorStored() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isStoreOffsetWithTermVector() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isStorePositionWithTermVector() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isBinary() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean getOmitNorms() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public void setOmitNorms(boolean omitNorms) {
|
||||
}
|
||||
|
||||
public boolean isLazy() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/////////////////////////////////////////////
|
||||
/// SolrInfoMBean
|
||||
|
|
|
@ -25,7 +25,6 @@ import java.net.MalformedURLException;
|
|||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
@ -59,13 +58,12 @@ import org.apache.solr.common.util.NamedList;
|
|||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.core.Config;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryResponse;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.search.SortSpec;
|
||||
import org.apache.solr.util.VersionedFile;
|
||||
import org.apache.solr.util.plugin.SolrCoreAware;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
|
@ -304,22 +302,22 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
|
|||
//---------------------------------------------------------------------------------
|
||||
|
||||
@Override
|
||||
public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
|
||||
public void prepare(ResponseBuilder rb) throws IOException
|
||||
{
|
||||
SolrQueryRequest req = rb.req;
|
||||
SolrParams params = req.getParams();
|
||||
// A runtime param can skip
|
||||
if( !params.getBool( ENABLE, true ) ) {
|
||||
return;
|
||||
}
|
||||
|
||||
ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
|
||||
Query query = builder.getQuery();
|
||||
Query query = rb.getQuery();
|
||||
if( query == null ) {
|
||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
|
||||
"The QueryElevationComponent needs to be registered 'after' the query component" );
|
||||
}
|
||||
|
||||
String qstr = getAnalyzedQuery( builder.getQueryString() );
|
||||
String qstr = getAnalyzedQuery( rb.getQueryString() );
|
||||
IndexReader reader = req.getSearcher().getReader();
|
||||
ElevationObj booster = null;
|
||||
try {
|
||||
|
@ -340,11 +338,11 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
|
|||
newq.add( bq );
|
||||
}
|
||||
}
|
||||
builder.setQuery( newq );
|
||||
rb.setQuery( newq );
|
||||
|
||||
// if the sort is 'score desc' use a custom sorting method to
|
||||
// insert documents in their proper place
|
||||
SortSpec sortSpec = builder.getSortSpec();
|
||||
SortSpec sortSpec = rb.getSortSpec();
|
||||
if( sortSpec.getSort() == null ) {
|
||||
sortSpec.setSort( new Sort( new SortField[] {
|
||||
new SortField(idField, new ElevationComparatorSource(booster.priority), false ),
|
||||
|
@ -377,7 +375,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
|
|||
}
|
||||
|
||||
// Add debugging information
|
||||
if( builder.isDebug() ) {
|
||||
if( rb.isDebug() ) {
|
||||
List<String> match = null;
|
||||
if( booster != null ) {
|
||||
// Extract the elevated terms into a list
|
||||
|
@ -391,12 +389,12 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
|
|||
SimpleOrderedMap<Object> dbg = new SimpleOrderedMap<Object>();
|
||||
dbg.add( "q", qstr );
|
||||
dbg.add( "match", match );
|
||||
builder.addDebugInfo( "queryBoosting", dbg );
|
||||
rb.addDebugInfo( "queryBoosting", dbg );
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
|
||||
public void process(ResponseBuilder rb) throws IOException {
|
||||
// Do nothing -- the real work is modifying the input query
|
||||
}
|
||||
|
||||
|
|
|
@ -18,23 +18,34 @@
|
|||
package org.apache.solr.handler.component;
|
||||
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.RTimer;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryResponse;
|
||||
import org.apache.solr.search.DocListAndSet;
|
||||
import org.apache.solr.search.QParser;
|
||||
import org.apache.solr.search.SortSpec;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* TODO!
|
||||
* This class is experimental and will be changing in the future.
|
||||
*
|
||||
* @version $Id$
|
||||
* @since solr 1.3
|
||||
*/
|
||||
public class ResponseBuilder
|
||||
{
|
||||
public SolrQueryRequest req;
|
||||
public SolrQueryResponse rsp;
|
||||
public boolean doHighlights;
|
||||
public boolean doFacets;
|
||||
|
||||
private boolean needDocList = false;
|
||||
private boolean needDocSet = false;
|
||||
private int fieldFlags = 0;
|
||||
|
@ -51,11 +62,74 @@ public class ResponseBuilder
|
|||
private RTimer timer = null;
|
||||
|
||||
private Query highlightQuery = null;
|
||||
|
||||
|
||||
//-------------------------------------------------------------------------
|
||||
//-------------------------------------------------------------------------
|
||||
|
||||
public List<SearchComponent> components;
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////
|
||||
//// Distributed Search section
|
||||
//////////////////////////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////
|
||||
|
||||
public static final String FIELD_SORT_VALUES = "fsv";
|
||||
public static final String SHARDS = "shards";
|
||||
public static final String IDS = "ids";
|
||||
|
||||
/***
|
||||
public static final String NUMDOCS = "nd";
|
||||
public static final String DOCFREQS = "tdf";
|
||||
public static final String TERMS = "terms";
|
||||
public static final String EXTRACT_QUERY_TERMS = "eqt";
|
||||
public static final String LOCAL_SHARD = "local";
|
||||
public static final String DOC_QUERY = "dq";
|
||||
***/
|
||||
|
||||
public static int STAGE_START = 0;
|
||||
public static int STAGE_PARSE_QUERY = 1000;
|
||||
public static int STAGE_EXECUTE_QUERY = 2000;
|
||||
public static int STAGE_GET_FIELDS = 3000;
|
||||
public static int STAGE_DONE = Integer.MAX_VALUE;
|
||||
|
||||
public int stage; // What stage is this current request at?
|
||||
|
||||
|
||||
public String[] shards;
|
||||
public List<ShardRequest> outgoing; // requests to be sent
|
||||
public List<ShardRequest> finished; // requests that have received responses from all shards
|
||||
|
||||
|
||||
public int getShardNum(String shard) {
|
||||
for (int i=0; i<shards.length; i++) {
|
||||
if (shards[i]==shard || shards[i].equals(shard)) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
public void addRequest(SearchComponent me, ShardRequest sreq) {
|
||||
outgoing.add(sreq);
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_PRIVATE)==0) {
|
||||
// if this isn't a private request, let other components modify it.
|
||||
for (SearchComponent component : components) {
|
||||
if (component != me) {
|
||||
component.modifyRequest(this, me, sreq);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public GlobalCollectionStat globalCollectionStat;
|
||||
|
||||
Map<Object, ShardDoc> resultIds;
|
||||
// Maps uniqueKeyValue to ShardDoc, which may be used to
|
||||
// determine order of the doc or uniqueKey in the final
|
||||
// returned sequence.
|
||||
// Only valid after STAGE_EXECUTE_QUERY has completed.
|
||||
|
||||
|
||||
/* private... components that don't own these shouldn't use them */
|
||||
SolrDocumentList _responseDocs;
|
||||
FacetInfo _facetInfo;
|
||||
|
||||
/**
|
||||
* Utility function to add debugging info. This will make sure a valid
|
||||
* debugInfo exists before adding to it.
|
||||
|
@ -70,7 +144,7 @@ public class ResponseBuilder
|
|||
|
||||
//-------------------------------------------------------------------------
|
||||
//-------------------------------------------------------------------------
|
||||
|
||||
|
||||
public boolean isDebug() {
|
||||
return debug;
|
||||
}
|
||||
|
@ -174,4 +248,17 @@ public class ResponseBuilder
|
|||
public void setTimer(RTimer timer) {
|
||||
this.timer = timer;
|
||||
}
|
||||
|
||||
|
||||
public static class GlobalCollectionStat {
|
||||
public final long numDocs;
|
||||
|
||||
public final Map<String, Long> dfMap;
|
||||
|
||||
public GlobalCollectionStat(int numDocs, Map<String, Long> dfMap) {
|
||||
this.numDocs = numDocs;
|
||||
this.dfMap = dfMap;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -36,11 +36,31 @@ import org.apache.solr.util.plugin.NamedListInitializedPlugin;
|
|||
*/
|
||||
public abstract class SearchComponent implements SolrInfoMBean, NamedListInitializedPlugin
|
||||
{
|
||||
public abstract void prepare( SolrQueryRequest req, SolrQueryResponse rsp ) throws IOException, ParseException;
|
||||
public abstract void process( SolrQueryRequest req, SolrQueryResponse rsp ) throws IOException;
|
||||
public abstract void prepare(ResponseBuilder rb) throws IOException;
|
||||
public abstract void process(ResponseBuilder rb) throws IOException;
|
||||
|
||||
/** Process for a distributed search.
|
||||
* @returns the next stage for this component */
|
||||
public int distributedProcess(ResponseBuilder rb) throws IOException {
|
||||
return ResponseBuilder.STAGE_DONE;
|
||||
}
|
||||
|
||||
/** Called after another component adds a request */
|
||||
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
|
||||
}
|
||||
|
||||
/** Called after all responses for a single request were received */
|
||||
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
|
||||
}
|
||||
|
||||
/** Called after all responses have been received for this stage.
|
||||
* Useful when different requests are sent to each shard.
|
||||
*/
|
||||
public void finishStage(ResponseBuilder rb) {
|
||||
}
|
||||
|
||||
|
||||
//////////////////////// NamedListInitializedPlugin methods //////////////////////
|
||||
|
||||
public void init( NamedList args )
|
||||
{
|
||||
// By default do nothing
|
||||
|
|
|
@ -17,41 +17,45 @@
|
|||
|
||||
package org.apache.solr.handler.component;
|
||||
|
||||
import org.apache.lucene.queryParser.ParseException;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.handler.RequestHandlerBase;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.RTimer;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.handler.RequestHandlerBase;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryResponse;
|
||||
import org.apache.solr.client.solrj.SolrServer;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.request.QueryRequest;
|
||||
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
|
||||
import org.apache.solr.util.plugin.SolrCoreAware;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.lucene.queryParser.ParseException;
|
||||
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
|
||||
import org.apache.commons.httpclient.HttpClient;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
|
||||
/**
|
||||
*
|
||||
* Refer SOLR-281
|
||||
*
|
||||
* @since solr 1.3
|
||||
*/
|
||||
public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
|
||||
{
|
||||
static final String RESPONSE_BUILDER_CONTEXT_KEY = "ResponseBuilder";
|
||||
|
||||
static final String INIT_COMPONENTS = "components";
|
||||
static final String INIT_FIRST_COMPONENTS = "first-components";
|
||||
static final String INIT_LAST_COMPONENTS = "last-components";
|
||||
|
||||
|
||||
protected static Logger log = Logger.getLogger(SearchHandler.class.getName());
|
||||
|
||||
|
||||
protected List<SearchComponent> components = null;
|
||||
protected NamedList initArgs = null;
|
||||
|
||||
|
||||
@Override
|
||||
public void init(NamedList args) {
|
||||
super.init( args );
|
||||
|
@ -73,7 +77,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
|
|||
* Initialize the components based on name
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public void inform(SolrCore core)
|
||||
public void inform(SolrCore core)
|
||||
{
|
||||
Object declaredComponents = initArgs.get(INIT_COMPONENTS);
|
||||
List<String> first = (List<String>) initArgs.get(INIT_FIRST_COMPONENTS);
|
||||
|
@ -83,13 +87,13 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
|
|||
if( declaredComponents == null ) {
|
||||
// Use the default component list
|
||||
list = getDefaultComponets();
|
||||
|
||||
|
||||
if( first != null ) {
|
||||
List<String> clist = first;
|
||||
clist.addAll( list );
|
||||
list = clist;
|
||||
}
|
||||
|
||||
|
||||
if( last != null ) {
|
||||
list.addAll( last );
|
||||
}
|
||||
|
@ -101,7 +105,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
|
|||
"First/Last components only valid if you do not declare 'components'");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Build the component list
|
||||
components = new ArrayList<SearchComponent>( list.size() );
|
||||
for(String c : list){
|
||||
|
@ -115,71 +119,147 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
|
|||
return components;
|
||||
}
|
||||
|
||||
public static ResponseBuilder getResponseBuilder(SolrQueryRequest req)
|
||||
{
|
||||
return (ResponseBuilder) req.getContext().get( RESPONSE_BUILDER_CONTEXT_KEY );
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------
|
||||
// SolrRequestHandler
|
||||
//---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@Override
|
||||
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, ParseException, InstantiationException, IllegalAccessException
|
||||
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception, ParseException, InstantiationException, IllegalAccessException
|
||||
{
|
||||
ResponseBuilder builder = new ResponseBuilder();
|
||||
req.getContext().put( RESPONSE_BUILDER_CONTEXT_KEY, builder );
|
||||
|
||||
if( components == null ) {
|
||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
|
||||
"SearchHandler not initialized properly. No components registered." );
|
||||
}
|
||||
|
||||
// The semantics of debugging vs not debugging are distinct enough
|
||||
// to justify two control loops
|
||||
if( !req.getParams().getBool( CommonParams.DEBUG_QUERY, false ) ) {
|
||||
// Prepare
|
||||
ResponseBuilder rb = new ResponseBuilder();
|
||||
rb.req = req;
|
||||
rb.rsp = rsp;
|
||||
rb.components = components;
|
||||
rb.setDebug(req.getParams().getBool(CommonParams.DEBUG_QUERY, false));
|
||||
|
||||
final RTimer timer = rb.isDebug() ? new RTimer() : null;
|
||||
|
||||
if (timer == null) {
|
||||
// non-debugging prepare phase
|
||||
for( SearchComponent c : components ) {
|
||||
c.prepare( req, rsp );
|
||||
c.prepare(rb);
|
||||
}
|
||||
|
||||
// Process
|
||||
for( SearchComponent c : components ) {
|
||||
c.process( req, rsp );
|
||||
}
|
||||
}
|
||||
else {
|
||||
builder.setDebug( true );
|
||||
RTimer timer = new RTimer();
|
||||
|
||||
// Prepare
|
||||
} else {
|
||||
// debugging prepare phase
|
||||
RTimer subt = timer.sub( "prepare" );
|
||||
for( SearchComponent c : components ) {
|
||||
builder.setTimer( subt.sub( c.getName() ) );
|
||||
c.prepare( req, rsp );
|
||||
builder.getTimer().stop();
|
||||
rb.setTimer( subt.sub( c.getName() ) );
|
||||
c.prepare(rb);
|
||||
rb.getTimer().stop();
|
||||
}
|
||||
subt.stop();
|
||||
|
||||
// Process
|
||||
subt = timer.sub( "process" );
|
||||
for( SearchComponent c : components ) {
|
||||
builder.setTimer( subt.sub( c.getName() ) );
|
||||
c.process( req, rsp );
|
||||
builder.getTimer().stop();
|
||||
}
|
||||
|
||||
if (rb.shards == null) {
|
||||
// a normal non-distributed request
|
||||
|
||||
// The semantics of debugging vs not debugging are different enough that
|
||||
// it makes sense to have two control loops
|
||||
if(!rb.isDebug()) {
|
||||
// Process
|
||||
for( SearchComponent c : components ) {
|
||||
c.process(rb);
|
||||
}
|
||||
}
|
||||
subt.stop();
|
||||
timer.stop();
|
||||
|
||||
// add the timing info
|
||||
builder.addDebugInfo( "timing", timer.asNamedList() );
|
||||
else {
|
||||
// Process
|
||||
RTimer subt = timer.sub( "process" );
|
||||
for( SearchComponent c : components ) {
|
||||
rb.setTimer( subt.sub( c.getName() ) );
|
||||
c.process(rb);
|
||||
rb.getTimer().stop();
|
||||
}
|
||||
subt.stop();
|
||||
timer.stop();
|
||||
|
||||
// add the timing info
|
||||
if( rb.getDebugInfo() == null ) {
|
||||
rb.setDebugInfo( new SimpleOrderedMap<Object>() );
|
||||
}
|
||||
rb.getDebugInfo().add( "timing", timer.asNamedList() );
|
||||
}
|
||||
|
||||
} else {
|
||||
// a distributed request
|
||||
|
||||
HttpCommComponent comm = new HttpCommComponent();
|
||||
|
||||
if (rb.outgoing == null) {
|
||||
rb.outgoing = new LinkedList<ShardRequest>();
|
||||
}
|
||||
rb.finished = new ArrayList<ShardRequest>();
|
||||
|
||||
int nextStage = 0;
|
||||
do {
|
||||
rb.stage = nextStage;
|
||||
nextStage = ResponseBuilder.STAGE_DONE;
|
||||
|
||||
// call all components
|
||||
for( SearchComponent c : components ) {
|
||||
// the next stage is the minimum of what all components report
|
||||
nextStage = Math.min(nextStage, c.distributedProcess(rb));
|
||||
}
|
||||
|
||||
|
||||
// check the outgoing queue and send requests
|
||||
while (rb.outgoing.size() > 0) {
|
||||
|
||||
// submit all current request tasks at once
|
||||
while (rb.outgoing.size() > 0) {
|
||||
ShardRequest sreq = rb.outgoing.remove(0);
|
||||
sreq.actualShards = sreq.shards;
|
||||
if (sreq.actualShards==ShardRequest.ALL_SHARDS) {
|
||||
sreq.actualShards = rb.shards;
|
||||
}
|
||||
sreq.responses = new ArrayList<ShardResponse>();
|
||||
|
||||
// TODO: map from shard to address[]
|
||||
for (String shard : sreq.actualShards) {
|
||||
ModifiableSolrParams params = sreq.params;
|
||||
params.remove("shards"); // not a top-level request
|
||||
params.remove("indent");
|
||||
params.remove("echoParams");
|
||||
params.set("isShard", true); // a sub (shard) request
|
||||
comm.submit(sreq, shard, params);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// now wait for replies, but if anyone puts more requests on
|
||||
// the outgoing queue, send them out immediately (by exiting
|
||||
// this loop)
|
||||
while (rb.outgoing.size() == 0) {
|
||||
ShardResponse srsp = comm.takeCompletedOrError();
|
||||
if (srsp == null) break; // no more requests to wait for
|
||||
|
||||
// Was there an exception? If so, abort everything and
|
||||
// rethrow
|
||||
if (srsp.exception != null) {
|
||||
comm.cancelAll();
|
||||
if (srsp.exception instanceof SolrException) {
|
||||
throw (SolrException)srsp.exception;
|
||||
} else {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, srsp.exception);
|
||||
}
|
||||
}
|
||||
|
||||
rb.finished.add(srsp.req);
|
||||
|
||||
// let the components see the responses to the request
|
||||
for(SearchComponent c : components) {
|
||||
c.handleResponses(rb, srsp.req);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for(SearchComponent c : components) {
|
||||
c.finishStage(rb);
|
||||
}
|
||||
|
||||
// we are done when the next stage is MAX_VALUE
|
||||
} while (nextStage != Integer.MAX_VALUE);
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------
|
||||
// SolrInfoMBeans
|
||||
//---------------------------------------------------------------------------------------
|
||||
|
||||
//////////////////////// SolrInfoMBeans methods //////////////////////
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
@ -206,3 +286,139 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
|
|||
return "$URL$";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// TODO: generalize how a comm component can fit into search component framework
|
||||
// TODO: statics should be per-core singletons
|
||||
|
||||
class HttpCommComponent {
|
||||
|
||||
// We want an executor that doesn't take up any resources if
|
||||
// it's not used, so it could be created statically for
|
||||
// the distributed search component if desired.
|
||||
//
|
||||
// Consider CallerRuns policy and a lower max threads to throttle
|
||||
// requests at some point (or should we simply return failure?)
|
||||
static Executor commExecutor = new ThreadPoolExecutor(
|
||||
0,
|
||||
Integer.MAX_VALUE,
|
||||
5, TimeUnit.SECONDS, // terminate idle threads after 5 sec
|
||||
new SynchronousQueue<Runnable>() // directly hand off tasks
|
||||
);
|
||||
|
||||
|
||||
static HttpClient client;
|
||||
|
||||
static {
|
||||
MultiThreadedHttpConnectionManager mgr = new MultiThreadedHttpConnectionManager();
|
||||
mgr.getParams().setDefaultMaxConnectionsPerHost(20);
|
||||
mgr.getParams().setMaxTotalConnections(10000);
|
||||
// mgr.getParams().setStaleCheckingEnabled(false);
|
||||
client = new HttpClient(mgr);
|
||||
}
|
||||
|
||||
CompletionService<ShardResponse> completionService = new ExecutorCompletionService<ShardResponse>(commExecutor);
|
||||
Set<Future<ShardResponse>> pending = new HashSet<Future<ShardResponse>>();
|
||||
|
||||
HttpCommComponent() {
|
||||
}
|
||||
|
||||
void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params) {
|
||||
Callable<ShardResponse> task = new Callable<ShardResponse>() {
|
||||
public ShardResponse call() throws Exception {
|
||||
|
||||
ShardResponse srsp = new ShardResponse();
|
||||
srsp.req = sreq;
|
||||
srsp.shard = shard;
|
||||
|
||||
try {
|
||||
// String url = "http://" + shard + "/select";
|
||||
String url = "http://" + shard;
|
||||
|
||||
params.remove("wt"); // use default (or should we explicitly set it?)
|
||||
params.remove("version");
|
||||
|
||||
SolrServer server = new CommonsHttpSolrServer(url, client);
|
||||
// SolrRequest req = new SolrRequest(SolrRequest.METHOD.GET, "/select");
|
||||
// use generic request to avoid extra processing of queries
|
||||
// QueryRequest req = new QueryRequest(sreq.params);
|
||||
// srsp.rsp = server.request(req);
|
||||
srsp.rsp = server.query(sreq.params);
|
||||
} catch (Throwable th) {
|
||||
srsp.exception = th;
|
||||
if (th instanceof SolrException) {
|
||||
srsp.rspCode = ((SolrException)th).code();
|
||||
} else {
|
||||
srsp.rspCode = -1;
|
||||
}
|
||||
}
|
||||
|
||||
return srsp;
|
||||
}
|
||||
};
|
||||
|
||||
pending.add( completionService.submit(task) );
|
||||
}
|
||||
|
||||
/** returns a ShardResponse of the last response correlated with a ShardRequest */
|
||||
ShardResponse take() {
|
||||
while (pending.size() > 0) {
|
||||
try {
|
||||
Future<ShardResponse> future = completionService.take();
|
||||
pending.remove(future);
|
||||
ShardResponse rsp = future.get();
|
||||
rsp.req.responses.add(rsp);
|
||||
if (rsp.req.responses.size() == rsp.req.actualShards.length) {
|
||||
return rsp;
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
|
||||
} catch (ExecutionException e) {
|
||||
// should be impossible... the problem with catching the exception
|
||||
// at this level is we don't know what ShardRequest it applied to
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Impossible Exception",e);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/** returns a ShardResponse of the last response correlated with a ShardRequest,
|
||||
* or immediately returns a ShardResponse if there was an error detected
|
||||
*/
|
||||
ShardResponse takeCompletedOrError() {
|
||||
while (pending.size() > 0) {
|
||||
try {
|
||||
Future<ShardResponse> future = completionService.take();
|
||||
pending.remove(future);
|
||||
ShardResponse rsp = future.get();
|
||||
if (rsp.exception != null) return rsp; // if exception, return immediately
|
||||
// add response to the response list... we do this after the take() and
|
||||
// not after the completion of "call" so we know when the last response
|
||||
// for a request was received. Otherwise we might return the same
|
||||
// request more than once.
|
||||
rsp.req.responses.add(rsp);
|
||||
if (rsp.req.responses.size() == rsp.req.actualShards.length) {
|
||||
return rsp;
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
|
||||
} catch (ExecutionException e) {
|
||||
// should be impossible... the problem with catching the exception
|
||||
// at this level is we don't know what ShardRequest it applied to
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Impossible Exception",e);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
void cancelAll() {
|
||||
for (Future<ShardResponse> future : pending) {
|
||||
// TODO: any issues with interrupting? shouldn't be if
|
||||
// there are finally blocks to release connections.
|
||||
future.cancel(true);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,269 @@
|
|||
package org.apache.solr.handler.component;
|
||||
|
||||
import org.apache.lucene.search.SortComparatorSource;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.apache.lucene.util.PriorityQueue;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.search.MissingStringLastComparatorSource;
|
||||
|
||||
import java.text.Collator;
|
||||
import java.util.Comparator;
|
||||
import java.util.Locale;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
public class ShardDoc {
|
||||
public String shard;
|
||||
public String shardAddress; // TODO
|
||||
|
||||
int orderInShard;
|
||||
// the position of this doc within the shard... this can be used
|
||||
// to short-circuit comparisons if the shard is equal, and can
|
||||
// also be used to break ties within the same shard.
|
||||
|
||||
Object id;
|
||||
// this is currently the uniqueKeyField but
|
||||
// may be replaced with internal docid in a future release.
|
||||
|
||||
Float score;
|
||||
|
||||
NamedList sortFieldValues;
|
||||
// sort field values for *all* docs in a particular shard.
|
||||
// this doc's values are in position orderInShard
|
||||
|
||||
// TODO: store the SolrDocument here?
|
||||
// Store the order in the merged list for lookup when getting stored fields?
|
||||
// (other components need this ordering to store data in order, like highlighting)
|
||||
// but we shouldn't expose uniqueKey (have a map by it) until the stored-field
|
||||
// retrieval stage.
|
||||
|
||||
int positionInResponse;
|
||||
// the ordinal position in the merged response arraylist
|
||||
|
||||
public String toString(){
|
||||
return "id="+id
|
||||
+" ,score="+score
|
||||
+" ,shard="+shard
|
||||
+" ,orderInShard="+orderInShard
|
||||
+" ,positionInResponse="+positionInResponse
|
||||
+" ,sortFieldValues="+sortFieldValues;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// used by distributed search to merge results.
|
||||
class ShardFieldSortedHitQueue extends PriorityQueue {
|
||||
|
||||
/** Stores a comparator corresponding to each field being sorted by */
|
||||
protected Comparator[] comparators;
|
||||
|
||||
/** Stores the sort criteria being used. */
|
||||
protected SortField[] fields;
|
||||
|
||||
/** The order of these fieldNames should correspond to the order of sort field values retrieved from the shard */
|
||||
protected List<String> fieldNames = new ArrayList<String>();
|
||||
|
||||
public ShardFieldSortedHitQueue(SortField[] fields, int size) {
|
||||
final int n = fields.length;
|
||||
comparators = new Comparator[n];
|
||||
this.fields = new SortField[n];
|
||||
for (int i = 0; i < n; ++i) {
|
||||
|
||||
// keep track of the named fields
|
||||
int type = fields[i].getType();
|
||||
if (type!=SortField.SCORE && type!=SortField.DOC) {
|
||||
fieldNames.add(fields[i].getField());
|
||||
}
|
||||
|
||||
String fieldname = fields[i].getField();
|
||||
comparators[i] = getCachedComparator(fieldname, fields[i]
|
||||
.getType(), fields[i].getLocale(), fields[i].getFactory());
|
||||
|
||||
if (fields[i].getType() == SortField.STRING) {
|
||||
this.fields[i] = new SortField(fieldname, fields[i].getLocale(),
|
||||
fields[i].getReverse());
|
||||
} else {
|
||||
this.fields[i] = new SortField(fieldname, fields[i].getType(),
|
||||
fields[i].getReverse());
|
||||
}
|
||||
|
||||
//System.out.println("%%%%%%%%%%%%%%%%%% got "+fields[i].getType() +" for "+ fieldname +" fields[i].getReverse(): "+fields[i].getReverse());
|
||||
}
|
||||
|
||||
initialize(size);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean lessThan(Object objA, Object objB) {
|
||||
ShardDoc docA = (ShardDoc)objA;
|
||||
ShardDoc docB = (ShardDoc)objB;
|
||||
|
||||
// If these docs are from the same shard, then the relative order
|
||||
// is how they appeared in the response from that shard.
|
||||
if (docA.shard == docB.shard) {
|
||||
// if docA has a smaller position, it should be "larger" so it
|
||||
// comes before docB.
|
||||
// This will handle sorting by docid within the same shard
|
||||
|
||||
// comment this out to test comparators.
|
||||
return !(docA.orderInShard < docB.orderInShard);
|
||||
}
|
||||
|
||||
|
||||
// run comparators
|
||||
final int n = comparators.length;
|
||||
int c = 0;
|
||||
for (int i = 0; i < n && c == 0; i++) {
|
||||
c = (fields[i].getReverse()) ? comparators[i].compare(docB, docA)
|
||||
: comparators[i].compare(docA, docB);
|
||||
}
|
||||
|
||||
// solve tiebreaks by comparing shards (similar to using docid)
|
||||
// smaller docid's beat larger ids, so reverse the natural ordering
|
||||
if (c == 0) {
|
||||
c = -docA.shard.compareTo(docB.shard);
|
||||
}
|
||||
|
||||
return c < 0;
|
||||
}
|
||||
|
||||
Comparator getCachedComparator(String fieldname, int type, Locale locale, SortComparatorSource factory) {
|
||||
Comparator comparator = null;
|
||||
switch (type) {
|
||||
case SortField.SCORE:
|
||||
comparator = comparatorScore(fieldname);
|
||||
break;
|
||||
case SortField.STRING:
|
||||
if (locale != null)
|
||||
comparator = comparatorStringLocale(fieldname, locale);
|
||||
else
|
||||
comparator = comparatorNatural(fieldname);
|
||||
break;
|
||||
case SortField.CUSTOM:
|
||||
if (factory instanceof MissingStringLastComparatorSource){
|
||||
comparator = comparatorMissingStringLast(fieldname);
|
||||
} else {
|
||||
// TODO: support other types such as random... is there a way to
|
||||
// support generically? Perhaps just comparing Object
|
||||
comparator = comparatorNatural(fieldname);
|
||||
// throw new RuntimeException("Custom sort not supported factory is "+factory.getClass());
|
||||
}
|
||||
break;
|
||||
case SortField.DOC:
|
||||
// TODO: we can support this!
|
||||
throw new RuntimeException("Doc sort not supported");
|
||||
default:
|
||||
comparator = comparatorNatural(fieldname);
|
||||
break;
|
||||
}
|
||||
return comparator;
|
||||
}
|
||||
|
||||
class ShardComparator implements Comparator {
|
||||
String fieldName;
|
||||
int fieldNum;
|
||||
public ShardComparator(String fieldName) {
|
||||
this.fieldName = fieldName;
|
||||
this.fieldNum=0;
|
||||
for (int i=0; i<fieldNames.size(); i++) {
|
||||
if (fieldNames.get(i).equals(fieldName)) {
|
||||
this.fieldNum = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Object sortVal(ShardDoc shardDoc) {
|
||||
assert(shardDoc.sortFieldValues.getName(fieldNum).equals(fieldName));
|
||||
List lst = (List)shardDoc.sortFieldValues.getVal(fieldNum);
|
||||
return lst.get(shardDoc.orderInShard);
|
||||
}
|
||||
|
||||
public int compare(Object o1, Object o2) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static Comparator comparatorScore(final String fieldName) {
|
||||
return new Comparator() {
|
||||
public final int compare(final Object o1, final Object o2) {
|
||||
ShardDoc e1 = (ShardDoc) o1;
|
||||
ShardDoc e2 = (ShardDoc) o2;
|
||||
|
||||
final float f1 = e1.score;
|
||||
final float f2 = e2.score;
|
||||
if (f1 < f2)
|
||||
return -1;
|
||||
if (f1 > f2)
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// The lucene natural sort ordering corresponds to numeric
|
||||
// and string natural sort orderings (ascending). Since
|
||||
// the PriorityQueue keeps the biggest elements by default,
|
||||
// we need to reverse the natural compare ordering so that the
|
||||
// smallest elements are kept instead of the largest... hence
|
||||
// the negative sign on the final compareTo().
|
||||
Comparator comparatorNatural(String fieldName) {
|
||||
return new ShardComparator(fieldName) {
|
||||
public final int compare(final Object o1, final Object o2) {
|
||||
ShardDoc sd1 = (ShardDoc) o1;
|
||||
ShardDoc sd2 = (ShardDoc) o2;
|
||||
Comparable v1 = (Comparable)sortVal(sd1);
|
||||
Comparable v2 = (Comparable)sortVal(sd2);
|
||||
if (v1==v2)
|
||||
return 0;
|
||||
if (v1==null)
|
||||
return 1;
|
||||
if(v2==null)
|
||||
return -1;
|
||||
return -v1.compareTo(v2);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Comparator comparatorStringLocale(final String fieldName,
|
||||
Locale locale) {
|
||||
final Collator collator = Collator.getInstance(locale);
|
||||
return new ShardComparator(fieldName) {
|
||||
public final int compare(final Object o1, final Object o2) {
|
||||
ShardDoc sd1 = (ShardDoc) o1;
|
||||
ShardDoc sd2 = (ShardDoc) o2;
|
||||
Comparable v1 = (Comparable)sortVal(sd1);
|
||||
Comparable v2 = (Comparable)sortVal(sd2);
|
||||
if (v1==v2)
|
||||
return 0;
|
||||
if (v1==null)
|
||||
return 1;
|
||||
if(v2==null)
|
||||
return -1;
|
||||
return -collator.compare(v1,v2);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Comparator comparatorMissingStringLast(final String fieldName) {
|
||||
return new ShardComparator(fieldName) {
|
||||
public final int compare(final Object o1, final Object o2) {
|
||||
ShardDoc sd1 = (ShardDoc) o1;
|
||||
ShardDoc sd2 = (ShardDoc) o2;
|
||||
Comparable v1 = (Comparable)sortVal(sd1);
|
||||
Comparable v2 = (Comparable)sortVal(sd2);
|
||||
if (v1==v2)
|
||||
return 0;
|
||||
if (v1==null)
|
||||
return -1;
|
||||
if(v2==null)
|
||||
return 1;
|
||||
return -v1.compareTo(v2);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,70 @@
|
|||
package org.apache.solr.handler.component;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrResponse;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
// todo... when finalized make accessors
|
||||
public class ShardRequest {
|
||||
public final static String[] ALL_SHARDS = null;
|
||||
|
||||
public final static int PURPOSE_PRIVATE = 0x01;
|
||||
public final static int PURPOSE_GET_TERM_DFS = 0x02;
|
||||
public final static int PURPOSE_GET_TOP_IDS = 0x04;
|
||||
public final static int PURPOSE_REFINE_TOP_IDS = 0x08;
|
||||
public final static int PURPOSE_GET_FACETS = 0x10;
|
||||
public final static int PURPOSE_REFINE_FACETS = 0x20;
|
||||
public final static int PURPOSE_GET_FIELDS = 0x40;
|
||||
public final static int PURPOSE_GET_HIGHLIGHTS = 0x80;
|
||||
public final static int PURPOSE_GET_DEBUG =0x100;
|
||||
|
||||
public int purpose; // the purpose of this request
|
||||
|
||||
public String[] shards; // the shards this request should be sent to, null for all
|
||||
// TODO: how to request a specific shard address?
|
||||
|
||||
|
||||
public ModifiableSolrParams params;
|
||||
|
||||
|
||||
/** list of responses... filled out by framework */
|
||||
public List<ShardResponse> responses = new ArrayList<ShardResponse>();
|
||||
|
||||
/** actual shards to send the request to, filled out by framework */
|
||||
public String[] actualShards;
|
||||
|
||||
// TODO: one could store a list of numbers to correlate where returned docs
|
||||
// go in the top-level response rather than looking up by id...
|
||||
// this would work well if we ever transitioned to using internal ids and
|
||||
// didn't require a uniqueId
|
||||
|
||||
public String toString() {
|
||||
return "ShardRequest:{params=" + params
|
||||
+ ", purpose=" + Integer.toHexString(purpose)
|
||||
+ ", nResponses =" + responses.size()
|
||||
+ "}";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ShardResponse {
|
||||
public ShardRequest req;
|
||||
public String shard;
|
||||
public String shardAddress; // the specific shard that this response was received from
|
||||
public int rspCode;
|
||||
public Throwable exception;
|
||||
public SolrResponse rsp;
|
||||
|
||||
public String toString() {
|
||||
return "ShardResponse:{shard="+shard+",shardAddress="+shardAddress
|
||||
+"\n\trequest=" + req
|
||||
+"\n\tresponse=" + rsp
|
||||
+ (exception==null ? "" : "\n\texception="+ SolrException.toStr(exception))
|
||||
+"\n}";
|
||||
}
|
||||
}
|
|
@ -62,12 +62,13 @@ public class SimpleFacets {
|
|||
protected SolrParams params;
|
||||
/** Searcher to use for all calculations */
|
||||
protected SolrIndexSearcher searcher;
|
||||
protected SolrQueryRequest req;
|
||||
|
||||
|
||||
public SimpleFacets(SolrIndexSearcher searcher,
|
||||
public SimpleFacets(SolrQueryRequest req,
|
||||
DocSet docs,
|
||||
SolrParams params) {
|
||||
this.searcher = searcher;
|
||||
this.req = req;
|
||||
this.searcher = req.getSearcher();
|
||||
this.docs = docs;
|
||||
this.params = params;
|
||||
}
|
||||
|
@ -117,12 +118,13 @@ public class SimpleFacets {
|
|||
* If user doesn't want schema default for facet.query, they should be
|
||||
* explicit.
|
||||
*/
|
||||
SolrQueryParser qp = searcher.getSchema().getSolrQueryParser(null);
|
||||
// SolrQueryParser qp = searcher.getSchema().getSolrQueryParser(null);
|
||||
|
||||
String[] facetQs = params.getParams(FacetParams.FACET_QUERY);
|
||||
if (null != facetQs && 0 != facetQs.length) {
|
||||
for (String q : facetQs) {
|
||||
res.add(q, searcher.numDocs(qp.parse(q), docs));
|
||||
Query qobj = QParser.getParser(q, null, req).getQuery();
|
||||
res.add(q, searcher.numDocs(qobj, docs));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -473,6 +473,7 @@ final public class XMLWriter {
|
|||
SolrIndexSearcher searcher = request.getSearcher();
|
||||
DocIterator iterator = ids.iterator();
|
||||
int sz = ids.size();
|
||||
includeScore = includeScore && ids.hasScores();
|
||||
for (int i=0; i<sz; i++) {
|
||||
int id = iterator.nextDoc();
|
||||
Document doc = searcher.doc(id, fields);
|
||||
|
|
|
@ -33,7 +33,7 @@ import java.io.IOException;
|
|||
*/
|
||||
|
||||
// move to apache package and make public if it is accepted as a patch
|
||||
class MissingStringLastComparatorSource implements SortComparatorSource {
|
||||
public class MissingStringLastComparatorSource implements SortComparatorSource {
|
||||
|
||||
public static final String bigString="\uffff\uffff\uffff\uffff\uffff\uffff\uffff\uffffNULL_VAL";
|
||||
|
||||
|
|
|
@ -197,7 +197,7 @@ public abstract class QParser {
|
|||
|
||||
QParserPlugin qplug = req.getCore().getQueryPlugin(type);
|
||||
return qplug.createParser(qstr, localParams, req.getParams(), req);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
@ -519,8 +519,15 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
|
|||
|
||||
private static Query matchAllDocsQuery = new MatchAllDocsQuery();
|
||||
|
||||
|
||||
protected DocSet getDocSet(List<Query> queries) throws IOException {
|
||||
/**
|
||||
* Returns the set of document ids matching all queries.
|
||||
* This method is cache-aware and attempts to retrieve the answer from the cache if possible.
|
||||
* If the answer was not cached, it may have been inserted into the cache as a result of this call.
|
||||
* This method can handle negative queries.
|
||||
* <p>
|
||||
* The DocSet returned should <b>not</b> be modified.
|
||||
*/
|
||||
public DocSet getDocSet(List<Query> queries) throws IOException {
|
||||
if (queries==null) return null;
|
||||
if (queries.size()==1) return getDocSet(queries.get(0));
|
||||
DocSet answer=null;
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
package org.apache.solr.search;
|
||||
|
||||
import org.apache.lucene.search.Sort;
|
||||
import org.apache.lucene.search.SortField;
|
||||
|
||||
/***
|
||||
* SortSpec encapsulates a Lucene Sort and a count of the number of documents
|
||||
|
@ -44,6 +45,18 @@ public class SortSpec
|
|||
sort = s;
|
||||
}
|
||||
|
||||
public static boolean includesScore(Sort sort) {
|
||||
if (sort==null) return true;
|
||||
for (SortField sf : sort.getSort()) {
|
||||
if (sf.getType() == SortField.SCORE) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean includesScore() {
|
||||
return includesScore(sort);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the Lucene Sort object, or null for the default sort
|
||||
* by score descending.
|
||||
|
|
|
@ -52,6 +52,6 @@ public class Sorting {
|
|||
}
|
||||
|
||||
|
||||
static final SortComparatorSource nullStringLastComparatorSource = new MissingStringLastComparatorSource();
|
||||
static final SortComparatorSource nullStringLastComparatorSource = new MissingStringLastComparatorSource(null);
|
||||
}
|
||||
|
||||
|
|
|
@ -437,11 +437,12 @@ public class SolrPluginUtils {
|
|||
|
||||
Document doc = searcher.doc(id);
|
||||
String strid = schema.printableUniqueKey(doc);
|
||||
String docname = "";
|
||||
if (strid != null) docname="id="+strid+",";
|
||||
docname = docname + "internal_docid="+id;
|
||||
|
||||
explainList.add(docname, "\n" +explain.toString());
|
||||
// String docname = "";
|
||||
// if (strid != null) docname="id="+strid+",";
|
||||
// docname = docname + "internal_docid="+id;
|
||||
|
||||
explainList.add(strid, "\n" +explain.toString());
|
||||
}
|
||||
return explainList;
|
||||
}
|
||||
|
|
|
@ -225,7 +225,7 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase {
|
|||
String resp = h.query(lrf.makeRequest("q", "text:hello", "debugQuery", "true"));
|
||||
//System.out.println(resp);
|
||||
// second doc ranked first
|
||||
assertTrue( resp.indexOf("id=2") < resp.indexOf("id=1") );
|
||||
assertTrue( resp.indexOf("\"2\"") < resp.indexOf("\"1\"") );
|
||||
}
|
||||
|
||||
public void testFieldBoost() throws Exception {
|
||||
|
@ -243,7 +243,7 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase {
|
|||
String resp = h.query(lrf.makeRequest("q", "text:hello", "debugQuery", "true"));
|
||||
//System.out.println(resp);
|
||||
// second doc ranked first
|
||||
assertTrue( resp.indexOf("id=2") < resp.indexOf("id=1") );
|
||||
assertTrue( resp.indexOf("\"2\"") < resp.indexOf("\"1\"") );
|
||||
}
|
||||
|
||||
public void testXMLWriter() throws Exception {
|
||||
|
|
|
@ -0,0 +1,504 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrServer;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.SolrResponse;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.client.solrj.request.QueryRequest;
|
||||
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
|
||||
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.util.AbstractSolrTestCase;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.*;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
/**
|
||||
* TODO? perhaps use:
|
||||
* http://docs.codehaus.org/display/JETTY/ServletTester
|
||||
* rather then open a real connection?
|
||||
*
|
||||
* @version $Id$
|
||||
* @since solr 1.3
|
||||
*/
|
||||
public class TestDistributedSearch extends TestCase {
|
||||
|
||||
Random r = new Random(0);
|
||||
File testDir;
|
||||
|
||||
int controlPort = 8985;
|
||||
SolrServer controlClient;
|
||||
JettySolrRunner controlJetty;
|
||||
|
||||
int[] ports = new int[] {7574, 7576};
|
||||
List<SolrServer> clients = new ArrayList<SolrServer>();
|
||||
List<JettySolrRunner> jettys = new ArrayList<JettySolrRunner>();
|
||||
String context = "/solr";
|
||||
String shards;
|
||||
|
||||
|
||||
String id="id";
|
||||
String t1="a_t";
|
||||
String i1="a_i";
|
||||
|
||||
|
||||
@Override public void setUp() throws Exception
|
||||
{
|
||||
System.setProperty("solr.test.sys.prop1", "propone");
|
||||
System.setProperty("solr.test.sys.prop2", "proptwo");
|
||||
testDir = new File(System.getProperty("java.io.tmpdir")
|
||||
+ System.getProperty("file.separator")
|
||||
+ getClass().getName() + "-" + System.currentTimeMillis());
|
||||
testDir.mkdirs();
|
||||
}
|
||||
|
||||
@Override public void tearDown() throws Exception
|
||||
{
|
||||
destroyServers();
|
||||
AbstractSolrTestCase.recurseDelete(testDir);
|
||||
}
|
||||
|
||||
|
||||
private void createServers() throws Exception {
|
||||
controlJetty = createJetty(controlPort);
|
||||
controlClient = createNewSolrServer(controlPort);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int port : ports) {
|
||||
if (sb.length()>0) sb.append(',');
|
||||
sb.append("localhost:"+port+context);
|
||||
jettys.add(createJetty(port));
|
||||
clients.add(createNewSolrServer(port));
|
||||
}
|
||||
|
||||
shards = sb.toString();
|
||||
}
|
||||
|
||||
private void destroyServers() throws Exception {
|
||||
controlJetty.stop();
|
||||
for (JettySolrRunner jetty : jettys) jetty.stop();
|
||||
clients.clear();
|
||||
jettys.clear();
|
||||
}
|
||||
|
||||
private JettySolrRunner createJetty(int port) throws Exception {
|
||||
File subDir = new File(testDir, ""+port);
|
||||
subDir.mkdirs();
|
||||
System.setProperty("solr.data.dir", subDir.toString());
|
||||
|
||||
JettySolrRunner jetty = new JettySolrRunner("/solr", port);
|
||||
jetty.start();
|
||||
return jetty;
|
||||
}
|
||||
|
||||
protected SolrServer createNewSolrServer(int port)
|
||||
{
|
||||
try {
|
||||
// setup the server...
|
||||
String url = "http://localhost:"+port+context;
|
||||
CommonsHttpSolrServer s = new CommonsHttpSolrServer( url );
|
||||
s.setConnectionTimeout(100); // 1/10th sec
|
||||
s.setDefaultMaxConnectionsPerHost(100);
|
||||
s.setMaxTotalConnections(100);
|
||||
return s;
|
||||
}
|
||||
catch( Exception ex ) {
|
||||
throw new RuntimeException( ex );
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void index(Object... fields) throws Exception {
|
||||
SolrInputDocument doc = new SolrInputDocument();
|
||||
for (int i=0; i<fields.length; i+=2) {
|
||||
doc.addField((String)(fields[i]), fields[i+1]);
|
||||
}
|
||||
controlClient.add(doc);
|
||||
|
||||
int which = (doc.getField(id).toString().hashCode() &0x7fffffff) % clients.size();
|
||||
SolrServer client = clients.get(which);
|
||||
client.add(doc);
|
||||
}
|
||||
|
||||
void index_specific(int serverNumber, Object... fields) throws Exception {
|
||||
SolrInputDocument doc = new SolrInputDocument();
|
||||
for (int i=0; i<fields.length; i+=2) {
|
||||
doc.addField((String)(fields[i]), fields[i+1]);
|
||||
}
|
||||
controlClient.add(doc);
|
||||
|
||||
int which = serverNumber;
|
||||
SolrServer client = clients.get(which);
|
||||
client.add(doc);
|
||||
}
|
||||
|
||||
void del(String q) throws Exception {
|
||||
controlClient.deleteByQuery(q);
|
||||
for (SolrServer client : clients) {
|
||||
client.deleteByQuery(q);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// serial commit...
|
||||
void commit() throws Exception {
|
||||
controlClient.commit();
|
||||
for (SolrServer client : clients) client.commit();
|
||||
}
|
||||
|
||||
void query(Object... q) throws Exception {
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
|
||||
for (int i=0; i<q.length; i+=2) {
|
||||
params.add(q[i].toString(), q[i+1].toString());
|
||||
}
|
||||
|
||||
QueryResponse controlRsp = controlClient.query(params);
|
||||
|
||||
// query a random server
|
||||
params.set("shards", shards);
|
||||
int which = r.nextInt(clients.size());
|
||||
SolrServer client = clients.get(which);
|
||||
QueryResponse rsp = client.query(params);
|
||||
|
||||
compareResponses(rsp, controlRsp);
|
||||
}
|
||||
|
||||
|
||||
private static int ORDERED=1;
|
||||
private static int SKIP=2;
|
||||
private static int SKIPVAL=4;
|
||||
private static int UNORDERED=8;
|
||||
|
||||
|
||||
public static boolean eq(String a, String b) {
|
||||
return a==b || (a != null && a.equals(b));
|
||||
}
|
||||
|
||||
public static int flags(Map<String,Integer> handle, Object key) {
|
||||
if (handle==null) return 0;
|
||||
Integer f = handle.get(key);
|
||||
return f == null ? 0 : f;
|
||||
}
|
||||
|
||||
public static String compare(NamedList a, NamedList b, int flags, Map<String,Integer> handle) {
|
||||
boolean ordered = (flags&UNORDERED) == 0;
|
||||
|
||||
int posa = 0, posb = 0;
|
||||
int na = 0, nb = 0;
|
||||
|
||||
for(;;) {
|
||||
if (posa >= a.size() || posb >= b.size()) {
|
||||
break;
|
||||
}
|
||||
|
||||
String namea=null, nameb=null;
|
||||
Object vala=null, valb=null;
|
||||
|
||||
int flagsa, flagsb;
|
||||
for (;;) {
|
||||
namea = a.getName(posa);
|
||||
vala = a.getVal(posa);
|
||||
posa++;
|
||||
flagsa = flags(handle, namea);
|
||||
if ((flagsa & SKIP) != 0) continue;
|
||||
na++;
|
||||
break;
|
||||
}
|
||||
|
||||
if (!ordered) posb=0; // reset if not ordered
|
||||
|
||||
while (posb<b.size()) {
|
||||
nameb = b.getName(posb);
|
||||
valb = b.getVal(posb);
|
||||
posb++;
|
||||
flagsb = flags(handle, nameb);
|
||||
if ((flagsb & SKIP) != 0) continue;
|
||||
if (eq(namea, nameb)) {
|
||||
nb++;
|
||||
break;
|
||||
}
|
||||
if (ordered) {
|
||||
return "."+namea+"!="+nameb+" (unordered or missing)";
|
||||
}
|
||||
// if unordered, continue until we find the right field.
|
||||
}
|
||||
|
||||
// ok, namea and nameb should be equal here already.
|
||||
if ((flagsa & SKIPVAL) != 0) continue; // keys matching is enough
|
||||
|
||||
String cmp = compare(vala, valb, flagsa, handle);
|
||||
if (cmp != null) return "."+namea+cmp;
|
||||
}
|
||||
|
||||
|
||||
if (na != nb) {
|
||||
return ".size()=="+na+","+nb;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static String compare1(Map a, Map b, int flags, Map<String,Integer> handle) {
|
||||
String cmp;
|
||||
|
||||
for (Object keya : a.keySet()) {
|
||||
Object vala = a.get(keya);
|
||||
int flagsa = flags(handle, keya);
|
||||
if ((flagsa & SKIP) != 0) continue;
|
||||
if (!b.containsKey(keya)) {
|
||||
return "[" + keya + "]==null";
|
||||
}
|
||||
if ((flagsa & SKIPVAL) != 0) continue;
|
||||
Object valb = b.get(keya);
|
||||
cmp = compare(vala, valb, flagsa, handle);
|
||||
if (cmp != null) return "[" + keya + "]" + cmp;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static String compare(Map a, Map b, int flags, Map<String,Integer> handle) {
|
||||
String cmp;
|
||||
cmp = compare1(a,b,flags,handle);
|
||||
if (cmp != null) return cmp;
|
||||
return compare1(b,a,flags,handle);
|
||||
}
|
||||
|
||||
public static String compare(SolrDocument a, SolrDocument b, int flags, Map<String,Integer> handle) {
|
||||
return compare(a.getFieldValuesMap(), b.getFieldValuesMap(), flags, handle);
|
||||
}
|
||||
|
||||
public static String compare(SolrDocumentList a, SolrDocumentList b, int flags, Map<String,Integer> handle) {
|
||||
boolean ordered = (flags & UNORDERED) == 0;
|
||||
|
||||
String cmp;
|
||||
int f = flags(handle, "maxScore");
|
||||
if ((f & SKIPVAL) == 0) {
|
||||
cmp = compare(a.getMaxScore(), b.getMaxScore(), 0, handle);
|
||||
if (cmp != null) return ".maxScore" + cmp;
|
||||
} else {
|
||||
if (a.getMaxScore() != null) {
|
||||
if (b.getMaxScore() == null) {
|
||||
return ".maxScore missing";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cmp = compare(a.getNumFound(), b.getNumFound(), 0, handle);
|
||||
if (cmp != null) return ".numFound" + cmp;
|
||||
|
||||
cmp = compare(a.getStart(), b.getStart(), 0, handle);
|
||||
if (cmp != null) return ".start" + cmp;
|
||||
|
||||
cmp = compare(a.size(), b.size(), 0, handle);
|
||||
if (cmp != null) return ".size()" + cmp;
|
||||
|
||||
// only for completely ordered results (ties might be in a different order)
|
||||
if (ordered) {
|
||||
for (int i=0; i<a.size(); i++) {
|
||||
cmp = compare(a.get(i), b.get(i), 0, handle);
|
||||
if (cmp != null) return "["+i+"]"+cmp;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// unordered case
|
||||
for (int i=0; i<a.size(); i++) {
|
||||
SolrDocument doc = a.get(i);
|
||||
Object key = doc.getFirstValue("id");
|
||||
SolrDocument docb=null;
|
||||
if (key==null) {
|
||||
// no id field to correlate... must compare ordered
|
||||
docb = b.get(i);
|
||||
} else {
|
||||
for (int j=0; j<b.size(); j++) {
|
||||
docb = b.get(j);
|
||||
if (key.equals(docb.getFirstValue("id"))) break;
|
||||
}
|
||||
}
|
||||
// if (docb == null) return "[id="+key+"]";
|
||||
cmp = compare(doc, docb, 0, handle);
|
||||
if (cmp != null) return "[id="+key+"]" + cmp;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static String compare(Object[] a, Object[] b, int flags, Map<String,Integer> handle) {
|
||||
if (a.length != b.length) {
|
||||
return ".length:"+a.length+"!="+b.length;
|
||||
}
|
||||
for (int i=0; i<a.length; i++) {
|
||||
String cmp = compare(a[i], b[i], flags, handle);
|
||||
if (cmp != null) return "["+i+"]"+cmp;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
static String compare(Object a, Object b, int flags, Map<String,Integer> handle) {
|
||||
if (a==b) return null;
|
||||
if (a==null || b==null) return ":" +a + "!=" + b;
|
||||
|
||||
if (a instanceof NamedList && b instanceof NamedList) {
|
||||
return compare((NamedList)a, (NamedList)b, flags, handle);
|
||||
}
|
||||
|
||||
if (a instanceof SolrDocumentList && b instanceof SolrDocumentList) {
|
||||
return compare((SolrDocumentList)a, (SolrDocumentList)b, flags, handle);
|
||||
}
|
||||
|
||||
if (a instanceof SolrDocument && b instanceof SolrDocument) {
|
||||
return compare((SolrDocument)a, (SolrDocument)b, flags, handle);
|
||||
}
|
||||
|
||||
if (a instanceof Map && b instanceof Map) {
|
||||
return compare((Map)a, (Map)b, flags, handle);
|
||||
}
|
||||
|
||||
if (a instanceof Object[] && b instanceof Object[]) {
|
||||
return compare((Object[])a, (Object[])b, flags, handle);
|
||||
}
|
||||
|
||||
if (!(a.equals(b))) {
|
||||
return ":" + a + "!=" + b;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
void compareResponses(QueryResponse a, QueryResponse b) {
|
||||
String cmp;
|
||||
System.out.println(a);
|
||||
System.out.println(b);
|
||||
|
||||
cmp = compare(a.getResponse(), b.getResponse(), flags, handle);
|
||||
if (cmp != null) {
|
||||
System.out.println(a);
|
||||
System.out.println(b);
|
||||
TestCase.fail(cmp);
|
||||
}
|
||||
}
|
||||
|
||||
int flags;
|
||||
Map<String, Integer> handle = new HashMap<String,Integer>();
|
||||
|
||||
|
||||
public void testDistribSearch() throws Exception {
|
||||
for (int nServers=1; nServers<4; nServers++) {
|
||||
ports = new int[nServers];
|
||||
for (int i=0; i<nServers; i++) {
|
||||
ports[i] = 7574 + i*2;
|
||||
}
|
||||
doTest();
|
||||
}
|
||||
}
|
||||
|
||||
public void doTest() throws Exception {
|
||||
createServers();
|
||||
del("*:*");
|
||||
index(id,1, i1, 100,t1,"now is the time for all good men");
|
||||
index(id,2, i1, 50 ,t1,"to come to the aid of their country.");
|
||||
index(id,3, i1, 2 ,t1,"how now brown cow");
|
||||
index(id,4, i1, -100 ,t1,"the quick fox jumped over the lazy dog");
|
||||
index(id,5, i1, 500 ,t1,"the quick fox jumped way over the lazy dog");
|
||||
index(id,6, i1, -600 ,t1,"humpty dumpy sat on a wall");
|
||||
index(id,7, i1, 123 ,t1,"humpty dumpy had a great fall");
|
||||
index(id,8, i1, 876 ,t1,"all the kings horses and all the kings men");
|
||||
index(id,9, i1, 7 ,t1,"couldn't put humpty together again");
|
||||
index(id,10, i1, 4321 ,t1,"this too shal pass");
|
||||
index(id,11, i1, -987 ,t1,"An eye for eye only ends up making the whole world blind.");
|
||||
index(id,12, i1, 379 ,t1,"Great works are performed, not by strength, but by perseverance.");
|
||||
|
||||
commit();
|
||||
|
||||
handle.clear();
|
||||
handle.put("QTime", SKIPVAL);
|
||||
handle.put("timestamp", SKIPVAL);
|
||||
|
||||
// these queries should be exactly ordered and scores should exactly match
|
||||
query("q","*:*", "sort",i1+" desc");
|
||||
query("q","<!func>"+i1);
|
||||
query("q","<!func>"+i1, "fl","*,score"); // even scores should match exactly here
|
||||
|
||||
handle.put("highlighting", UNORDERED);
|
||||
handle.put("response", UNORDERED);
|
||||
|
||||
query("q","quick");
|
||||
query("q","all","fl","id","start","0");
|
||||
query("q","all","fl","foofoofoo","start","0"); // no fields in returned docs
|
||||
query("q","all","fl","id","start","100");
|
||||
|
||||
handle.put("maxScore", SKIPVAL);
|
||||
handle.put("score", SKIPVAL);
|
||||
query("q","quick","fl","*,score");
|
||||
query("q","all","fl","*,score","start","1");
|
||||
query("q","all","fl","*,score","start","100");
|
||||
|
||||
query("q","now their fox sat had put","fl","*,score",
|
||||
"hl","true","hl.fl",t1);
|
||||
|
||||
query("q","now their fox sat had put","fl","foofoofoo",
|
||||
"hl","true","hl.fl",t1);
|
||||
|
||||
|
||||
handle.put("debug", UNORDERED);
|
||||
handle.put("time", SKIPVAL);
|
||||
|
||||
query("q","now their fox sat had put","fl","*,score",
|
||||
"debugQuery", "true");
|
||||
|
||||
query("q","*:*", "rows",100, "facet","true", "facet.field",t1);
|
||||
query("q","*:*", "rows",100, "facet","true", "facet.query","quick", "facet.query","all", "facet.query","*:*");
|
||||
query("q","*:*", "rows",100, "facet","true", "facet.field",t1, "facet.offset",1);
|
||||
query("q","*:*", "rows",100, "facet","true", "facet.field",t1,"facet.mincount",2);
|
||||
|
||||
|
||||
// index the same document to two servers and make sure things
|
||||
// don't blow up.
|
||||
if (clients.size()>=2) {
|
||||
index(id,100, i1, 107 ,t1,"oh no, a duplicate!");
|
||||
for (int i=0; i<clients.size(); i++) {
|
||||
index_specific(i, id,100, i1, 107 ,t1,"oh no, a duplicate!");
|
||||
}
|
||||
commit();
|
||||
query("q","duplicate", "hl","true", "hl.fl", t1);
|
||||
query("q","fox duplicate horses", "hl","true", "hl.fl", t1);
|
||||
query("q","*:*", "rows",100);
|
||||
}
|
||||
|
||||
// Thread.sleep(10000000000L);
|
||||
|
||||
destroyServers();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -27,9 +27,7 @@
|
|||
<!-- Used to specify an alternate directory to hold all index data.
|
||||
It defaults to "index" if not present, and should probably
|
||||
not be changed if replication is in use. -->
|
||||
<!--
|
||||
<indexDir>index</indexDir>
|
||||
-->
|
||||
<dataDir>${solr.data.dir:./solr/data}</dataDir>
|
||||
|
||||
<indexDefaults>
|
||||
<!-- Values here affect all index writers and act as a default
|
||||
|
|
Loading…
Reference in New Issue