From fe2135963c250643cabb85864a74116fb38e57ed Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Wed, 13 May 2020 09:56:53 -0400 Subject: [PATCH] SOLR-14407: Handle shards.purpose in the postlogs tool --- .../org/apache/solr/util/SolrLogPostTool.java | 71 ++++++++++++++++++- .../apache/solr/util/SolrLogPostToolTest.java | 6 +- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java index 249146d4b21..a1b67dd2920 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java +++ b/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java @@ -18,9 +18,12 @@ package org.apache.solr.util; import java.io.*; import java.nio.charset.Charset; +import java.util.Collections; import java.util.List; import java.util.ArrayList; import java.net.URLDecoder; +import java.util.Map; +import java.util.TreeMap; import java.util.UUID; import java.util.regex.Pattern; import java.util.regex.Matcher; @@ -29,6 +32,8 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; +import org.apache.solr.handler.component.ShardRequest; + /** @@ -309,6 +314,7 @@ public class SolrLogPostTool { return doc; } + private SolrInputDocument parseNewSearch(String line) { SolrInputDocument doc = new SolrInputDocument(); @@ -492,7 +498,7 @@ public class SolrLogPostTool { doc.addField("shards_s", "true"); } - if(parts[0].equals("ids") && ! isRTGRequest(doc)) { + if(parts[0].equals("ids") && !isRTGRequest(doc)) { doc.addField("ids_s", "true"); } @@ -510,13 +516,25 @@ public class SolrLogPostTool { String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); doc.addField("facet_s", dr); } - } + if(parts[0].equals("shards.purpose")) { + try { + int purpose = Integer.parseInt(parts[1]); + String[] purposes = getRequestPurposeNames(purpose); + for (String p : purposes) { + doc.addField("purpose_ss", p); + } + } catch(Throwable e) { + //We'll just sit on this for now and not interrupt the load for this one field. + } + } + } //Special params used to determine what stage a query is. //So we populate with defaults. //The absence of the distrib params means its a distributed query. + if(doc.getField("distrib_s") == null) { doc.addField("distrib_s", "true"); } @@ -538,4 +556,53 @@ public class SolrLogPostTool { return "/get".equals(path.getValue()); } } + + private static final Map purposes; + protected static final String UNKNOWN_VALUE = "Unknown"; + private static final String[] purposeUnknown = new String[] { UNKNOWN_VALUE }; + + public static String[] getRequestPurposeNames(Integer reqPurpose) { + if (reqPurpose != null) { + int valid = 0; + for (Map.Entryentry : purposes.entrySet()) { + if ((reqPurpose & entry.getKey()) != 0) { + valid++; + } + } + if (valid == 0) { + return purposeUnknown; + } else { + String[] result = new String[valid]; + int i = 0; + for (Map.Entryentry : purposes.entrySet()) { + if ((reqPurpose & entry.getKey()) != 0) { + result[i] = entry.getValue(); + i++; + } + } + return result; + } + } + return purposeUnknown; + } + + static { + Map map = new TreeMap<>(); + map.put(ShardRequest.PURPOSE_PRIVATE, "PRIVATE"); + map.put(ShardRequest.PURPOSE_GET_TOP_IDS, "GET_TOP_IDS"); + map.put(ShardRequest.PURPOSE_REFINE_TOP_IDS, "REFINE_TOP_IDS"); + map.put(ShardRequest.PURPOSE_GET_FACETS, "GET_FACETS"); + map.put(ShardRequest.PURPOSE_REFINE_FACETS, "REFINE_FACETS"); + map.put(ShardRequest.PURPOSE_GET_FIELDS, "GET_FIELDS"); + map.put(ShardRequest.PURPOSE_GET_HIGHLIGHTS, "GET_HIGHLIGHTS"); + map.put(ShardRequest.PURPOSE_GET_DEBUG, "GET_DEBUG"); + map.put(ShardRequest.PURPOSE_GET_STATS, "GET_STATS"); + map.put(ShardRequest.PURPOSE_GET_TERMS, "GET_TERMS"); + map.put(ShardRequest.PURPOSE_GET_TOP_GROUPS, "GET_TOP_GROUPS"); + map.put(ShardRequest.PURPOSE_GET_MLT_RESULTS, "GET_MLT_RESULTS"); + map.put(ShardRequest.PURPOSE_REFINE_PIVOT_FACETS, "REFINE_PIVOT_FACETS"); + map.put(ShardRequest.PURPOSE_SET_TERM_STATS, "SET_TERM_STATS"); + map.put(ShardRequest.PURPOSE_GET_TERM_STATS, "GET_TERM_STATS"); + purposes = Collections.unmodifiableMap(map); + } } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java b/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java index 9f9322a974d..10f7b8c734a 100644 --- a/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java +++ b/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java @@ -33,7 +33,7 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 { @Test public void testQueryRecord() throws Exception{ - String record = "2019-12-09 15:05:01.931 INFO (qtp2103763750-21) [c:logs4 s:shard1 r:core_node2 x:logs4_shard1_replica_n1] o.a.s.c.S.Request [logs4_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_=1575835181759&isShard=true&wt=javabin&distrib=false} hits=234868 status=0 QTime=8\n"; + String record = "2019-12-09 15:05:01.931 INFO (qtp2103763750-21) [c:logs4 s:shard1 r:core_node2 x:logs4_shard1_replica_n1] o.a.s.c.S.Request [logs4_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_=1575835181759&shards.purpose=36&isShard=true&wt=javabin&distrib=false} hits=234868 status=0 QTime=8\n"; List docs = readDocs(record); assertEquals(docs.size(), 1); SolrInputDocument doc = docs.get(0); @@ -53,6 +53,8 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 { SolrInputField isShard = doc.getField("isShard_s"); SolrInputField ids = doc.getField("ids_s"); SolrInputField shards = doc.getField("shards_s"); + SolrInputField purpose = doc.getField("purpose_ss"); + Object[] purposes = purpose.getValues().toArray(); assertEquals(query.getValue(), "*:*"); assertEquals(date.getValue(), "2019-12-09T15:05:01.931"); @@ -69,6 +71,8 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 { assertEquals(isShard.getValue(), "true"); assertEquals(ids.getValue(), "false"); assertEquals(shards.getValue(), "false"); + assertEquals("GET_TOP_IDS", purposes[0].toString()); + assertEquals("REFINE_FACETS", purposes[1].toString()); } @Test