From d1abb0961257c704299b9a62ae33fb796ba91832 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 23 Feb 2012 09:07:31 +0000 Subject: [PATCH] SOLR-3134: Include shard info in distributed response when shards.info=true git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1292717 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 4 +- .../handler/component/HttpShardHandler.java | 42 +++++++++-------- .../handler/component/QueryComponent.java | 18 ++++++++ .../solr/handler/component/SearchHandler.java | 45 ++++++++++++++++--- .../apache/solr/TestDistributedSearch.java | 21 +++++++++ .../solr/common/params/ShardParams.java | 6 +++ 6 files changed, 109 insertions(+), 27 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 5bf4409c3a1..4c37a3f77c5 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -1,4 +1,4 @@ - Apache Solr Release Notes + Apache Solr Release Notes Introduction ------------ @@ -227,6 +227,8 @@ New Features * SOLR-2459: Expose LogLevel selection with a RequestHandler rather then servlet (Stefan Matheis, Upayavira, ryan) +* SOLR-3134: Include shard info in distributed response when shards.info=true (ryan) + Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index 90790f9b176..d50e96c1d15 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -39,12 +39,13 @@ import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; -import org.apache.solr.core.CoreDescriptor; -import org.apache.solr.request.SolrQueryRequest; - -import java.util.*; -import java.util.concurrent.*; - +import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.request.SolrQueryRequest; + +import java.net.ConnectException; +import java.util.*; +import java.util.concurrent.*; + public class HttpShardHandler extends ShardHandler { private HttpShardHandlerFactory httpShardHandlerFactory; @@ -155,12 +156,15 @@ public class HttpShardHandler extends ShardHandler { ssr.nl = server.request(req); } else { LBHttpSolrServer.Rsp rsp = httpShardHandlerFactory.loadbalancer.request(new LBHttpSolrServer.Req(req, urls)); - ssr.nl = rsp.getResponse(); - srsp.setShardAddress(rsp.getServer()); - } - } catch (Throwable th) { - srsp.setException(th); - if (th instanceof SolrException) { + ssr.nl = rsp.getResponse(); + srsp.setShardAddress(rsp.getServer()); + } + } + catch( ConnectException cex ) { + srsp.setException(cex); //???? + } catch (Throwable th) { + srsp.setException(th); + if (th instanceof SolrException) { srsp.setResponseCode(((SolrException)th).code()); } else { srsp.setResponseCode(-1); @@ -245,13 +249,13 @@ public class HttpShardHandler extends ShardHandler { String shards = params.get(ShardParams.SHARDS); // for back compat, a shards param with URLs like localhost:8983/solr will mean that this - // search is distributed. - boolean hasShardURL = shards != null && shards.indexOf('/') > 0; - rb.isDistrib = hasShardURL | rb.isDistrib; - - if (rb.isDistrib) { - // since the cost of grabbing cloud state is still up in the air, we grab it only - // if we need it. + // search is distributed. + boolean hasShardURL = shards != null && shards.indexOf('/') > 0; + rb.isDistrib = hasShardURL | rb.isDistrib; + + if (rb.isDistrib) { + // since the cost of grabbing cloud state is still up in the air, we grab it only + // if we need it. CloudState cloudState = null; Map slices = null; CoreDescriptor coreDescriptor = req.getCore().getCoreDescriptor(); diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 73e9cf0bc60..60368b1acb4 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -44,6 +44,7 @@ import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.*; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.request.SolrQueryRequest; @@ -768,11 +769,28 @@ public class QueryComponent extends SearchComponent ShardFieldSortedHitQueue queue; queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount()); + NamedList shardInfo = null; + if(rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) { + shardInfo = (NamedList) rb.rsp.getValues().get(ShardParams.SHARDS_INFO); + if(shardInfo==null) { + shardInfo = new SimpleOrderedMap(); + rb.rsp.getValues().add(ShardParams.SHARDS_INFO,shardInfo); + } + } + long numFound = 0; Float maxScore=null; for (ShardResponse srsp : sreq.responses) { SolrDocumentList docs = (SolrDocumentList)srsp.getSolrResponse().getResponse().get("response"); + if(shardInfo!=null) { + SimpleOrderedMap nl = new SimpleOrderedMap(); + nl.add("numFound", docs.getNumFound()); + nl.add("maxScore", docs.getMaxScore()); + nl.add("time", srsp.getSolrResponse().getElapsedTime()); + shardInfo.add(srsp.getShard(), nl); + } + // calculate global maxScore and numDocsFound if (docs.getMaxScore() != null) { maxScore = maxScore==null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore()); diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index 94a43fefc68..ea6cd0878b5 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -18,11 +18,14 @@ package org.apache.solr.handler.component; import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.RTimer; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.CloseHook; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; @@ -35,6 +38,8 @@ import org.apache.solr.util.plugin.SolrCoreAware; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.*; /** @@ -283,14 +288,40 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware , ShardResponse srsp = shardHandler1.takeCompletedOrError(); if (srsp == null) break; // no more requests to wait for - // Was there an exception? If so, abort everything and - // rethrow + // Was there an exception? if (srsp.getException() != null) { - shardHandler1.cancelAll(); - if (srsp.getException() instanceof SolrException) { - throw (SolrException)srsp.getException(); - } else { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, srsp.getException()); + // If things are tolerant, just continue + if(rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false)) { + if( rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false) ) { + NamedList sinfo = (NamedList) rb.rsp.getValues().get(ShardParams.SHARDS_INFO); + if(sinfo==null) { + sinfo = new SimpleOrderedMap(); + rb.rsp.getValues().add(ShardParams.SHARDS_INFO,sinfo); + } + + SimpleOrderedMap nl = new SimpleOrderedMap(); + Throwable t = srsp.getException(); + if(t instanceof SolrServerException) { + t = ((SolrServerException)t).getCause(); + } + nl.add("error", t.toString() ); + + StringWriter trace = new StringWriter(); + t.printStackTrace(new PrintWriter(trace)); + nl.add("trace", trace.toString() ); + if(srsp.getSolrResponse()!=null){ + nl.add("time", srsp.getSolrResponse().getElapsedTime()); + } + sinfo.add(srsp.getShard(), nl); + } + } + else { // If so, abort everything and rethrow + shardHandler1.cancelAll(); + if (srsp.getException() instanceof SolrException) { + throw (SolrException)srsp.getException(); + } else { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, srsp.getException()); + } } } diff --git a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java index 479b8aa5720..a5053d5c822 100755 --- a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java +++ b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java @@ -17,7 +17,12 @@ package org.apache.solr; +import org.apache.commons.lang.StringUtils; +import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.util.NamedList; /** * TODO? perhaps use: @@ -274,6 +279,22 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS); query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY); + // Check Info is added to for each shard + ModifiableSolrParams q = new ModifiableSolrParams(); + q.set("q", "*:*"); + q.set(ShardParams.SHARDS_INFO, true); + setDistributedParams(q); + QueryResponse rsp = queryServer(q); + NamedList sinfo = (NamedList) rsp.getResponse().get(ShardParams.SHARDS_INFO); + String shards = getShardsString(); + int cnt = StringUtils.countMatches(shards, ",")+1; + + assertNotNull("missing shard info", sinfo); + assertEquals("should have an entry for each shard ["+sinfo+"] "+shards, cnt, sinfo.size()); + + + // This index has the same number for every field + // TODO: This test currently fails because debug info is obtained only // on shards with matches. // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); diff --git a/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java b/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java index 3af8a962d22..a71e6299283 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java @@ -39,4 +39,10 @@ public interface ShardParams { /** query type for shard requests */ public static final String SHARDS_QT = "shards.qt"; + + /** Request detailed match info for each shard (true/false) */ + public static final String SHARDS_INFO = "shards.info"; + + /** Should things fail if there is an error? (true/false) */ + public static final String SHARDS_TOLERANT = "shards.tolerant"; }