diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 040b7147b11..4699fcf3210 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -69,6 +69,11 @@ Upgrading from Solr 4.6.0 Detailed Change List ---------------------- +Other Changes +--------------------- + +* SOLR-5399: Add distributed request tracking information to DebugComponent + (Tomás Fernández Löbbe via Ryan Ernst) ================== 4.6.0 ================== diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 6983a75b0dd..fe3c455be23 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -17,27 +17,6 @@ package org.apache.solr.core; -import com.google.common.collect.Maps; - -import org.apache.solr.cloud.ZkController; -import org.apache.solr.cloud.ZkSolrResourceLoader; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.ZooKeeperException; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.SolrjNamedThreadFactory; -import org.apache.solr.handler.admin.CollectionsHandler; -import org.apache.solr.handler.admin.CoreAdminHandler; -import org.apache.solr.handler.admin.InfoHandler; -import org.apache.solr.handler.component.ShardHandlerFactory; -import org.apache.solr.logging.LogWatcher; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.IndexSchemaFactory; -import org.apache.solr.util.DefaultSolrThreadFactory; -import org.apache.solr.util.FileUtils; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static com.google.common.base.Preconditions.checkNotNull; import java.io.File; @@ -67,6 +46,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.ZooKeeperException; import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.handler.admin.CoreAdminHandler; import org.apache.solr.handler.admin.InfoHandler; @@ -120,6 +100,8 @@ public class CoreContainer { protected final String solrHome; protected final CoresLocator coresLocator; + + private String hostName; { log.info("New CoreContainer " + System.identityHashCode(this)); @@ -224,6 +206,9 @@ public class CoreContainer { if (shareSchema) { indexSchemaCache = new ConcurrentHashMap(); } + + hostName = cfg.getHost(); + log.info("Host Name: " + hostName); zkSys.initZooKeeper(this, solrHome, cfg); @@ -899,6 +884,10 @@ public class CoreContainer { public String getAdminPath() { return cfg.getAdminPath(); } + + public String getHostName() { + return this.hostName; + } /** * Gets the alternate path for multicore handling: diff --git a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java index ff97632caf5..238d6740dd8 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java @@ -19,15 +19,26 @@ package org.apache.solr.handler.component; import static org.apache.solr.common.params.CommonParams.FQ; -import org.apache.solr.common.params.CommonParams; - import java.io.IOException; import java.net.URL; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.search.Query; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.QueryParsing; import org.apache.solr.util.SolrPluginUtils; @@ -41,10 +52,34 @@ public class DebugComponent extends SearchComponent { public static final String COMPONENT_NAME = "debug"; + /** + * A counter to ensure that no RID is equal, even if they fall in the same millisecond + */ + private static final AtomicLong ridCounter = new AtomicLong(); + + /** + * Map containing all the possible stages as key and + * the corresponding readable purpose as value + */ + private static final Map stages; + + static { + Map map = new TreeMap<>(); + map.put(ResponseBuilder.STAGE_START, "START"); + map.put(ResponseBuilder.STAGE_PARSE_QUERY, "PARSE_QUERY"); + map.put(ResponseBuilder.STAGE_TOP_GROUPS, "TOP_GROUPS"); + map.put(ResponseBuilder.STAGE_EXECUTE_QUERY, "EXECUTE_QUERY"); + map.put(ResponseBuilder.STAGE_GET_FIELDS, "GET_FIELDS"); + map.put(ResponseBuilder.STAGE_DONE, "DONE"); + stages = Collections.unmodifiableMap(map); + } + @Override public void prepare(ResponseBuilder rb) throws IOException { - + if(rb.isDebugTrack() && rb.isDistrib) { + doDebugTrack(rb); + } } @SuppressWarnings("unchecked") @@ -52,7 +87,6 @@ public class DebugComponent extends SearchComponent public void process(ResponseBuilder rb) throws IOException { if( rb.isDebug() ) { - NamedList stdinfo = SolrPluginUtils.doStandardDebug( rb.req, rb.getQueryString(), rb.getQuery(), rb.getResults().docList, rb.isDebugQuery(), rb.isDebugResults()); @@ -68,7 +102,7 @@ public class DebugComponent extends SearchComponent if (rb.isDebugQuery() && rb.getQparser() != null) { rb.getQparser().addDebugInfo(rb.getDebugInfo()); } - + if (null != rb.getDebugInfo() ) { if (rb.isDebugQuery() && null != rb.getFilters() ) { info.add("filter_queries",rb.req.getParams().getParams(FQ)); @@ -86,29 +120,68 @@ public class DebugComponent extends SearchComponent } + private void doDebugTrack(ResponseBuilder rb) { + SolrQueryRequest req = rb.req; + String rid = req.getParams().get(CommonParams.REQUEST_ID); + if(rid == null || "".equals(rid)) { + rid = generateRid(rb); + ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); + params.add(CommonParams.REQUEST_ID, rid);//add rid to the request so that shards see it + req.setParams(params); + } + rb.addDebug(rid, "track", CommonParams.REQUEST_ID);//to see it in the response + rb.rsp.addToLog(CommonParams.REQUEST_ID, rid); //to see it in the logs of the landing core + + } + + private String generateRid(ResponseBuilder rb) { + String hostName = rb.req.getCore().getCoreDescriptor().getCoreContainer().getHostName(); + return hostName + "-" + rb.req.getCore().getName() + "-" + System.currentTimeMillis() + "-" + ridCounter.getAndIncrement(); + } + @Override public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) { if (!rb.isDebug()) return; - + // Turn on debug to get explain only when retrieving fields if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) { sreq.purpose |= ShardRequest.PURPOSE_GET_DEBUG; if (rb.isDebugAll()) { sreq.params.set(CommonParams.DEBUG_QUERY, "true"); - } else if (rb.isDebugQuery()){ - sreq.params.set(CommonParams.DEBUG, CommonParams.QUERY); - } else if (rb.isDebugTimings()){ - sreq.params.set(CommonParams.DEBUG, CommonParams.TIMING); - } else if (rb.isDebugResults()){ - sreq.params.set(CommonParams.DEBUG, CommonParams.RESULTS); + } else if (rb.isDebug()) { + if (rb.isDebugQuery()){ + sreq.params.add(CommonParams.DEBUG, CommonParams.QUERY); + } + if (rb.isDebugTimings()){ + sreq.params.add(CommonParams.DEBUG, CommonParams.TIMING); + } + if (rb.isDebugResults()){ + sreq.params.add(CommonParams.DEBUG, CommonParams.RESULTS); + } } } else { sreq.params.set(CommonParams.DEBUG_QUERY, "false"); } + if (rb.isDebugTrack()) { + sreq.params.add(CommonParams.DEBUG, CommonParams.TRACK); + sreq.params.set(CommonParams.REQUEST_ID, rb.req.getParams().get(CommonParams.REQUEST_ID)); + sreq.params.set(CommonParams.REQUEST_PURPOSE, SolrPluginUtils.getRequestPurpose(sreq.purpose)); + } } @Override public void handleResponses(ResponseBuilder rb, ShardRequest sreq) { + if (rb.isDebugTrack() && rb.isDistrib && !rb.finished.isEmpty()) { + @SuppressWarnings("unchecked") + NamedList stageList = (NamedList) ((NamedList)rb.getDebugInfo().get("track")).get(stages.get(rb.stage)); + if(stageList == null) { + stageList = new NamedList(); + rb.addDebug(stageList, "track", stages.get(rb.stage)); + } + for(ShardResponse response: sreq.responses) { + stageList.add(response.getShard(), getTrackResponse(response)); + } + } } private Set excludeSet = new HashSet(Arrays.asList("explain")); @@ -116,7 +189,7 @@ public class DebugComponent extends SearchComponent @Override public void finishStage(ResponseBuilder rb) { if (rb.isDebug() && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) { - NamedList info = null; + NamedList info = rb.getDebugInfo(); NamedList explain = new SimpleOrderedMap(); Map.Entry[] arr = new NamedList.NamedListEntry[rb.resultIds.size()]; @@ -165,9 +238,28 @@ public class DebugComponent extends SearchComponent rb.setDebugInfo(info); rb.rsp.add("debug", rb.getDebugInfo() ); } + } + private NamedList getTrackResponse(ShardResponse shardResponse) { + NamedList namedList = new NamedList(); + NamedList responseNL = shardResponse.getSolrResponse().getResponse(); + @SuppressWarnings("unchecked") + NamedList responseHeader = (NamedList)responseNL.get("responseHeader"); + if(responseHeader != null) { + namedList.add("QTime", responseHeader.get("QTime").toString()); + } + namedList.add("ElapsedTime", String.valueOf(shardResponse.getSolrResponse().getElapsedTime())); + namedList.add("RequestPurpose", shardResponse.getShardRequest().params.get(CommonParams.REQUEST_PURPOSE)); + SolrDocumentList docList = (SolrDocumentList)shardResponse.getSolrResponse().getResponse().get("response"); + if(docList != null) { + namedList.add("NumFound", String.valueOf(docList.getNumFound())); + } + namedList.add("Response", String.valueOf(responseNL)); + return namedList; + } + Object merge(Object source, Object dest, Set exclude) { if (source == null) return dest; if (dest == null) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java index 8f00c421162..ac1712faf1d 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java @@ -62,7 +62,7 @@ public class ResponseBuilder private boolean needDocSet = false; private int fieldFlags = 0; //private boolean debug = false; - private boolean debugTimings, debugQuery, debugResults; + private boolean debugTimings, debugQuery, debugResults, debugTrack; private QParser qparser = null; private String queryString = null; @@ -212,7 +212,7 @@ public class ResponseBuilder //------------------------------------------------------------------------- public boolean isDebug() { - return debugQuery || debugTimings || debugResults; + return debugQuery || debugTimings || debugResults || debugTrack; } /** @@ -220,13 +220,22 @@ public class ResponseBuilder * @return true if all debugging options are on */ public boolean isDebugAll(){ - return debugQuery && debugTimings && debugResults; + return debugQuery && debugTimings && debugResults && debugTrack; } - + public void setDebug(boolean dbg){ debugQuery = dbg; debugTimings = dbg; debugResults = dbg; + debugTrack = dbg; + } + + public boolean isDebugTrack() { + return debugTrack; + } + + public void setDebugTrack(boolean debugTrack) { + this.debugTrack = debugTrack; } public boolean isDebugTimings() { diff --git a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java index 18e24dfdcf7..ee2bb2495fd 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java +++ b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java @@ -17,6 +17,20 @@ package org.apache.solr.util; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Pattern; + import org.apache.lucene.index.StorableField; import org.apache.lucene.index.StoredDocument; import org.apache.lucene.search.BooleanClause; @@ -37,6 +51,7 @@ import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.component.HighlightComponent; import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.handler.component.ShardRequest; import org.apache.solr.highlight.SolrHighlighter; import org.apache.solr.parser.QueryParser; import org.apache.solr.request.SolrQueryRequest; @@ -56,18 +71,6 @@ import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrQueryParser; import org.apache.solr.search.SyntaxError; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; -import java.lang.reflect.Method; -import java.lang.reflect.InvocationTargetException; - /** *

Utilities that may be of use to RequestHandlers.

* @@ -82,6 +85,31 @@ import java.lang.reflect.InvocationTargetException; * default parameter settings. */ public class SolrPluginUtils { + + + /** + * Map containing all the possible purposes codes of a request as key and + * the corresponding readable purpose as value + */ + private static final Map purposes; + + static { + Map map = new TreeMap<>(); + map.put(ShardRequest.PURPOSE_PRIVATE, "PRIVATE"); + map.put(ShardRequest.PURPOSE_GET_TERM_DFS, "GET_TERM_DFS"); + map.put(ShardRequest.PURPOSE_GET_TOP_IDS, "GET_TOP_IDS"); + map.put(ShardRequest.PURPOSE_REFINE_TOP_IDS, "REFINE_TOP_IDS"); + map.put(ShardRequest.PURPOSE_GET_FACETS, "GET_FACETS"); + map.put(ShardRequest.PURPOSE_REFINE_FACETS, "REFINE_FACETS"); + map.put(ShardRequest.PURPOSE_GET_FIELDS, "GET_FIELDS"); + map.put(ShardRequest.PURPOSE_GET_HIGHLIGHTS, "GET_HIGHLIGHTS"); + map.put(ShardRequest.PURPOSE_GET_DEBUG, "GET_DEBUG"); + map.put(ShardRequest.PURPOSE_GET_STATS, "GET_STATS"); + map.put(ShardRequest.PURPOSE_GET_TERMS, "GET_TERMS"); + map.put(ShardRequest.PURPOSE_GET_TOP_GROUPS, "GET_TOP_GROUPS"); + map.put(ShardRequest.PURPOSE_GET_MLT_RESULTS, "GET_MLT_RESULTS"); + purposes = Collections.unmodifiableMap(map); + } /** * Set default-ish params on a SolrQueryRequest. @@ -204,6 +232,8 @@ public class SolrPluginUtils { rb.setDebugQuery(true); } else if (params[i].equals(CommonParams.RESULTS)){ rb.setDebugResults(true); + } else if (params[i].equals(CommonParams.TRACK)){ + rb.setDebugTrack(true); } } } @@ -262,6 +292,7 @@ public class SolrPluginUtils { return dbg; } + public static void doStandardQueryDebug( SolrQueryRequest req, String userQuery, @@ -299,7 +330,6 @@ public class SolrPluginUtils { dbg.add("explain", explainStruct ? explanationsToNamedLists(explain) : explanationsToStrings(explain)); - String otherQueryS = req.getParams().get(CommonParams.EXPLAIN_OTHER); if (otherQueryS != null && otherQueryS.length() > 0) { DocList otherResults = doSimpleQuery(otherQueryS, req, 0, 10); @@ -642,6 +672,7 @@ public class SolrPluginUtils { // Pattern to detect consecutive + and/or - operators // \s+[+-](?:\s*[+-]+)+ private final static Pattern CONSECUTIVE_OP_PATTERN = Pattern.compile( "\\s+[+-](?:\\s*[+-]+)+" ); + protected static final String UNKNOWN_VALUE = "Unknown"; /** * Strips operators that are used illegally, otherwise returns its @@ -930,6 +961,33 @@ public class SolrPluginUtils { } } } + + + + /** + * Given the integer purpose of a request generates a readable value corresponding + * the request purposes (there can be more than one on a single request). If + * there is a purpose parameter present that's not known this method will + * return {@value #UNKNOWN_VALUE} + * @param reqPurpose Numeric request purpose + * @return a comma separated list of purposes or {@value #UNKNOWN_VALUE} + */ + public static String getRequestPurpose(Integer reqPurpose) { + if (reqPurpose != null) { + StringBuilder builder = new StringBuilder(); + for (Map.Entryentry : purposes.entrySet()) { + if ((reqPurpose & entry.getKey()) != 0) { + builder.append(entry.getValue() + ","); + } + } + if (builder.length() == 0) { + return UNKNOWN_VALUE; + } + builder.setLength(builder.length() - 1); + return builder.toString(); + } + return UNKNOWN_VALUE; + } } diff --git a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java index bb8de0705ba..d18dde208c4 100644 --- a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java +++ b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java @@ -113,6 +113,8 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { vals[i] = "test " + i; } indexr(id, 17, "SubjectTerms_mfacet", vals); + + for (int i=100; i<150; i++) { indexr(id, i); @@ -315,6 +317,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { handle.put("explain", SKIPVAL); // internal docids differ, idf differs w/o global idf handle.put("debug", UNORDERED); handle.put("time", SKIPVAL); + handle.put("track", SKIP); //track is not included in single node search query("q","now their fox sat had put","fl","*,score",CommonParams.DEBUG_QUERY, "true"); query("q", "id:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING); diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java index 43d073d0389..fe71e88889b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java @@ -516,6 +516,7 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase { handle.put("explain", SKIPVAL); handle.put("debug", UNORDERED); handle.put("time", SKIPVAL); + handle.put("track", SKIP); query("q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true"); query("q", "id:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 279047abbde..032987187f4 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -326,6 +326,7 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase { handle.put("explain", SKIPVAL); handle.put("debug", UNORDERED); handle.put("time", SKIPVAL); + handle.put("track", SKIP); query(false, new Object[] {"q","now their fox sat had put","fl","*,score",CommonParams.DEBUG_QUERY, "true"}); query(false, new Object[] {"q", "id:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"}); query(false, new Object[] {"q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING}); diff --git a/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java index 6f205586532..d0ba84beeeb 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java @@ -16,8 +16,18 @@ package org.apache.solr.handler.component; * limitations under the License. */ +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; + import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; import org.junit.BeforeClass; import org.junit.Test; @@ -139,4 +149,90 @@ public class DebugComponentTest extends SolrTestCaseJ4 { ); } + + @Test + public void testModifyRequestTrack() { + DebugComponent component = new DebugComponent(); + List components = new ArrayList(1); + components.add(component); + for(int i = 0; i < 10; i++) { + SolrQueryRequest req = req("q", "test query", "distrib", "true", CommonParams.REQUEST_ID, "123456-my_rid"); + SolrQueryResponse resp = new SolrQueryResponse(); + ResponseBuilder rb = new ResponseBuilder(req, resp, components); + ShardRequest sreq = new ShardRequest(); + sreq.params = new ModifiableSolrParams(); + sreq.purpose = ShardRequest.PURPOSE_GET_FIELDS; + sreq.purpose |= ShardRequest.PURPOSE_GET_DEBUG; + //expecting the same results with debugQuery=true or debug=track + if(random().nextBoolean()) { + rb.setDebug(true); + } else { + rb.setDebug(false); + rb.setDebugTrack(true); + //should not depend on other debug options + rb.setDebugQuery(random().nextBoolean()); + rb.setDebugTimings(random().nextBoolean()); + rb.setDebugResults(random().nextBoolean()); + } + component.modifyRequest(rb, null, sreq); + //if the request has debugQuery=true or debug=track, the sreq should get debug=track always + assertTrue(Arrays.asList(sreq.params.getParams(CommonParams.DEBUG)).contains(CommonParams.TRACK)); + //the purpose must be added as readable param to be included in the shard logs + assertEquals("GET_FIELDS,GET_DEBUG", sreq.params.get(CommonParams.REQUEST_PURPOSE)); + //the rid must be added to be included in the shard logs + assertEquals("123456-my_rid", sreq.params.get(CommonParams.REQUEST_ID)); + } + + } + + @Test + public void testPrepare() throws IOException { + DebugComponent component = new DebugComponent(); + List components = new ArrayList(1); + components.add(component); + SolrQueryRequest req; + ResponseBuilder rb; + for(int i = 0; i < 10; i++) { + req = req("q", "test query", "distrib", "true"); + rb = new ResponseBuilder(req, new SolrQueryResponse(), components); + rb.isDistrib = true; + //expecting the same results with debugQuery=true or debug=track + if(random().nextBoolean()) { + rb.setDebug(true); + } else { + rb.setDebug(false); + rb.setDebugTrack(true); + //should not depend on other debug options + rb.setDebugQuery(random().nextBoolean()); + rb.setDebugTimings(random().nextBoolean()); + rb.setDebugResults(random().nextBoolean()); + } + component.prepare(rb); + ensureRidPresent(rb, null); + } + + req = req("q", "test query", "distrib", "true", CommonParams.REQUEST_ID, "123"); + rb = new ResponseBuilder(req, new SolrQueryResponse(), components); + rb.isDistrib = true; + rb.setDebug(true); + component.prepare(rb); + ensureRidPresent(rb, "123"); + } + + @SuppressWarnings("unchecked") + private void ensureRidPresent(ResponseBuilder rb, String expectedRid) { + SolrQueryRequest req = rb.req; + SolrQueryResponse resp = rb.rsp; + //a generated request ID should be added to the request + String rid = req.getParams().get(CommonParams.REQUEST_ID); + if(expectedRid == null) { + assertTrue(rid + " Doesn't match expected pattern.", Pattern.matches(".*-collection1-[0-9]*-[0-9]+", rid)); + } else { + assertEquals("Expecting " + expectedRid + " but found " + rid, expectedRid, rid); + } + //The request ID is added to the debug/track section + assertEquals(rid, ((NamedList)rb.getDebugInfo().get("track")).get(CommonParams.REQUEST_ID)); + //RID must be added to the toLog, so that it's included in the main request log + assertEquals(rid, resp.getToLog().get(CommonParams.REQUEST_ID)); + } } diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java new file mode 100644 index 00000000000..381e6859c92 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java @@ -0,0 +1,151 @@ +package org.apache.solr.handler.component; + +import java.io.File; + +import org.apache.commons.io.FileUtils; +import org.apache.solr.SolrJettyTestBase; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.util.NamedList; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class DistributedDebugComponentTest extends SolrJettyTestBase { + + private static SolrServer collection1; + private static SolrServer collection2; + private static String shard1; + private static String shard2; + private static File solrHome; + + @BeforeClass + public static void beforeTest() throws Exception { + solrHome = createSolrHome(); + createJetty(solrHome.getAbsolutePath(), null, null); + + String url = jetty.getBaseUrl().toString(); + collection1 = new HttpSolrServer(url); + collection2 = new HttpSolrServer(url + "/collection2"); + + String urlCollection1 = jetty.getBaseUrl().toString() + "/" + "collection1"; + String urlCollection2 = jetty.getBaseUrl().toString() + "/" + "collection2"; + shard1 = urlCollection1.replaceAll("http://", ""); + shard2 = urlCollection2.replaceAll("http://", ""); + + //create second core + CoreAdminRequest.Create req = new CoreAdminRequest.Create(); + req.setCoreName("collection2"); + collection1.request(req); + } + + private static File createSolrHome() throws Exception { + File workDir = new File(System.getProperty("tempDir", System.getProperty("java.io.tmpdir"))); + workDir = new File(workDir, DistributedDebugComponentTest.class.getName()); + setupJettyTestHome(workDir, "collection1"); + FileUtils.copyDirectory(new File(workDir, "collection1"), new File(workDir, "collection2")); + return workDir; + } + + @AfterClass + public static void afterTest() throws Exception { + cleanUpJettyHome(solrHome); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + + SolrInputDocument doc = new SolrInputDocument(); + doc.setField("id", "1"); + doc.setField("text", "batman"); + collection1.add(doc); + collection1.commit(); + + doc.setField("id", "2"); + doc.setField("text", "superman"); + collection2.add(doc); + collection2.commit(); + + } + + @Test + @SuppressWarnings("unchecked") + public void testSimpleSearch() throws Exception { + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.set("debug", "track"); + query.set("distrib", "true"); + query.setFields("id", "text"); + query.set("shards", shard1 + "," + shard2); + QueryResponse response = collection1.query(query); + NamedList track = (NamedList) response.getDebugMap().get("track"); + assertNotNull(track); + assertNotNull(track.get("rid")); + assertNotNull(track.get("EXECUTE_QUERY")); + assertNotNull(((NamedList)track.get("EXECUTE_QUERY")).get(shard1)); + assertNotNull(((NamedList)track.get("EXECUTE_QUERY")).get(shard2)); + + assertNotNull(((NamedList)track.get("GET_FIELDS")).get(shard1)); + assertNotNull(((NamedList)track.get("GET_FIELDS")).get(shard2)); + + assertElementsPresent((NamedList)((NamedList)track.get("EXECUTE_QUERY")).get(shard1), + "QTime", "ElapsedTime", "RequestPurpose", "NumFound", "Response"); + assertElementsPresent((NamedList)((NamedList)track.get("EXECUTE_QUERY")).get(shard2), + "QTime", "ElapsedTime", "RequestPurpose", "NumFound", "Response"); + + assertElementsPresent((NamedList)((NamedList)track.get("GET_FIELDS")).get(shard1), + "QTime", "ElapsedTime", "RequestPurpose", "NumFound", "Response"); + assertElementsPresent((NamedList)((NamedList)track.get("GET_FIELDS")).get(shard2), + "QTime", "ElapsedTime", "RequestPurpose", "NumFound", "Response"); + + query.add("omitHeader", "true"); + response = collection1.query(query); + assertNull("QTime is not included in the response when omitHeader is set to true", + ((NamedList)response.getDebugMap().get("track")).findRecursive("EXECUTE_QUERY", shard1, "QTime")); + assertNull("QTime is not included in the response when omitHeader is set to true", + ((NamedList)response.getDebugMap().get("track")).findRecursive("GET_FIELDS", shard2, "QTime")); + + query.setQuery("id:1"); + response = collection1.query(query); + track = (NamedList) response.getDebugMap().get("track"); + assertNotNull(((NamedList)track.get("EXECUTE_QUERY")).get(shard1)); + assertNotNull(((NamedList)track.get("EXECUTE_QUERY")).get(shard2)); + + assertNotNull(((NamedList)track.get("GET_FIELDS")).get(shard1)); + // This test is invalid, as GET_FIELDS should not be executed in shard 2 + assertNull(((NamedList)track.get("GET_FIELDS")).get(shard2)); + } + + private void assertElementsPresent(NamedList namedList, String...elements) { + for(String element:elements) { + String value = namedList.get(element); + assertNotNull("Expected element '" + element + "' but was not found", value); + assertTrue("Expected element '" + element + "' but was empty", !value.isEmpty()); + } + } + +} diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java index 9f14c89dc8f..b7aee35f189 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java @@ -122,7 +122,10 @@ public interface CommonParams { * {@link #DEBUG} value indicating an interest in debug output related to the Query (parsing, etc.) */ public static final String QUERY = "query"; - + /** + * {@link #DEBUG} value indicating an interest in debug output related to the distributed tracking + */ + public static final String TRACK = "track"; /** * boolean indicating whether score explanations should structured (true), * or plain text (false) @@ -206,5 +209,15 @@ public interface CommonParams { */ public static final String COST = "cost"; + /** + * Request ID parameter added to the request when using debug=track + */ + public static final String REQUEST_ID = "rid"; + + /** + * Request Purpose parameter added to each internal shard request when using debug=track + */ + public static final String REQUEST_PURPOSE = "requestPurpose"; + }