Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-14558: Record all log lines in SolrLogPostTool (#1570)

This commit is contained in:
Jason Gerlowski 2020-06-16 08:21:24 -04:00 committed by GitHub
parent a108f90869
commit a7792b129b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 71 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
View File

@ -186,20 +186,25 @@ public class SolrLogPostTool {
} }
if (line != null) { if (line != null) {
SolrInputDocument lineDoc = new SolrInputDocument();
lineDoc.setField("date_dt", parseDate(line));
lineDoc.setField("line_t", line);
lineDoc.setField("type_s", "other"); // Overridden by known types below
if (line.contains("Registered new searcher")) { if (line.contains("Registered new searcher")) {
return parseNewSearch(line); parseNewSearch(lineDoc, line);
} else if (line.contains("path=/update")) { } else if (line.contains("path=/update")) {
return parseUpdate(line); parseUpdate(lineDoc, line);
} else if (line.contains(" ERROR ")) { } else if (line.contains(" ERROR ")) {
this.cause = null; this.cause = null;
return parseError(line, readTrace()); parseError(lineDoc, line, readTrace());
} else if (line.contains("start commit")) { } else if (line.contains("start commit")) {
return parseCommit(line); parseCommit(lineDoc, line);
} else if(line.contains("QTime=")) { } else if(line.contains("QTime=")) {
return parseQueryRecord(line); parseQueryRecord(lineDoc, line);
} else {
continue;
} }
return lineDoc;
} else { } else {
return null; return null;
} }
@ -252,94 +257,73 @@ public class SolrLogPostTool {
doc.setField(fieldName, fieldValue); doc.setField(fieldName, fieldValue);
} }
private SolrInputDocument parseError(String line, String trace) throws IOException { private void parseError(SolrInputDocument lineRecord, String line, String trace) throws IOException {
SolrInputDocument doc = new SolrInputDocument(); lineRecord.setField("type_s", "error");
doc.setField("date_dt", parseDate(line));
doc.setField("type_s", "error");
doc.setField("line_t", line);
//Don't include traces that have only the %html header. //Don't include traces that have only the %html header.
if(trace != null && trace.length() > 6) { if(trace != null && trace.length() > 6) {
doc.setField("stack_t", trace); lineRecord.setField("stack_t", trace);
} }
if(this.cause != null) { if(this.cause != null) {
doc.setField("root_cause_t", cause.replace("Caused by:", "").trim()); lineRecord.setField("root_cause_t", cause.replace("Caused by:", "").trim());
} }
doc.setField("collection_s", parseCollection(line)); lineRecord.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line)); lineRecord.setField("core_s", parseCore(line));
doc.setField("shard_s", parseShard(line)); lineRecord.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line)); lineRecord.setField("replica_s", parseReplica(line));
return doc;
} }
private SolrInputDocument parseCommit(String line) throws IOException { private void parseCommit(SolrInputDocument lineRecord, String line) throws IOException {
SolrInputDocument doc = new SolrInputDocument(); lineRecord.setField("type_s", "commit");
doc.setField("date_dt", parseDate(line)); lineRecord.setField("soft_commit_s", Boolean.toString(line.contains("softCommit=true")));
doc.setField("type_s", "commit");
doc.setField("line_t", line);
doc.setField("soft_commit_s", Boolean.toString(line.contains("softCommit=true")));
doc.setField("open_searcher_s", Boolean.toString(line.contains("openSearcher=true"))); lineRecord.setField("open_searcher_s", Boolean.toString(line.contains("openSearcher=true")));
doc.setField("collection_s", parseCollection(line)); lineRecord.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line)); lineRecord.setField("core_s", parseCore(line));
doc.setField("shard_s", parseShard(line)); lineRecord.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line)); lineRecord.setField("replica_s", parseReplica(line));
return doc;
} }
private SolrInputDocument parseQueryRecord(String line) { private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
lineRecord.setField("qtime_i", parseQTime(line));
SolrInputDocument doc = new SolrInputDocument(); lineRecord.setField("status_s", parseStatus(line));
doc.setField("date_dt", parseDate(line));
doc.setField("qtime_i", parseQTime(line));
doc.setField("status_s", parseStatus(line));
String path = parsePath(line); String path = parsePath(line);
doc.setField("path_s", path); lineRecord.setField("path_s", path);
if(line.contains("hits=")) { if(line.contains("hits=")) {
doc.setField("hits_l", parseHits(line)); lineRecord.setField("hits_l", parseHits(line));
} }
String params = parseParams(line); String params = parseParams(line);
doc.setField("params_t", params); lineRecord.setField("params_t", params);
addParams(doc, params); addParams(lineRecord, params);
doc.setField("collection_s", parseCollection(line)); lineRecord.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line)); lineRecord.setField("core_s", parseCore(line));
doc.setField("node_s", parseNode(line)); lineRecord.setField("node_s", parseNode(line));
doc.setField("shard_s", parseShard(line)); lineRecord.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line)); lineRecord.setField("replica_s", parseReplica(line));
if(path != null && path.contains("/admin")) { if(path != null && path.contains("/admin")) {
doc.setField("type_s", "admin"); lineRecord.setField("type_s", "admin");
} else if(path != null && params.contains("/replication")) { } else if(path != null && params.contains("/replication")) {
doc.setField("type_s", "replication"); lineRecord.setField("type_s", "replication");
} else if (path != null && path.contains("/get")) { } else if (path != null && path.contains("/get")) {
doc.setField("type_s", "get"); lineRecord.setField("type_s", "get");
} else { } else {
doc.setField("type_s", "query"); lineRecord.setField("type_s", "query");
} }
return doc;
} }
private SolrInputDocument parseNewSearch(String line) { private void parseNewSearch(SolrInputDocument lineRecord, String line) {
lineRecord.setField("core_s", parseNewSearcherCore(line));
SolrInputDocument doc = new SolrInputDocument(); lineRecord.setField("type_s", "newSearcher");
doc.setField("date_dt", parseDate(line));
doc.setField("core_s", parseNewSearcherCore(line));
doc.setField("type_s", "newSearcher");
doc.setField("line_t", line);
return doc;
} }
private String parseCollection(String line) { private String parseCollection(String line) {
@ -352,25 +336,19 @@ public class SolrLogPostTool {
} }
} }
private SolrInputDocument parseUpdate(String line) { private void parseUpdate(SolrInputDocument lineRecord, String line) {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));
if(line.contains("deleteByQuery=")) { if(line.contains("deleteByQuery=")) {
doc.setField("type_s", "deleteByQuery"); lineRecord.setField("type_s", "deleteByQuery");
} else if(line.contains("delete=")) { } else if(line.contains("delete=")) {
doc.setField("type_s", "delete"); lineRecord.setField("type_s", "delete");
} else { } else {
doc.setField("type_s", "update"); lineRecord.setField("type_s", "update");
} }
doc.setField("collection_s", parseCollection(line)); lineRecord.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line)); lineRecord.setField("core_s", parseCore(line));
doc.setField("shard_s", parseShard(line)); lineRecord.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line)); lineRecord.setField("replica_s", parseReplica(line));
doc.setField("line_t", line);
return doc;
} }
private String parseNewSearcherCore(String line) { private String parseNewSearcherCore(String line) {

16
solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
View File

@ -18,6 +18,7 @@ package org.apache.solr.util;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.StringReader; import java.io.StringReader;
import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.ArrayList; import java.util.ArrayList;
@ -286,6 +287,21 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
assertEquals(core.getValue(), "production_cv_month_201912_shard35_replica_n1"); assertEquals(core.getValue(), "production_cv_month_201912_shard35_replica_n1");
} }
// Ensure SolrLogPostTool parses _all_ log lines into searchable records
@Test
public void testOtherRecord() throws Exception {
final String record = "2020-06-11 11:59:08.386 INFO (main) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)";
final List<SolrInputDocument> docs = readDocs(record);
assertEquals(docs.size(), 1);
SolrInputDocument doc = docs.get(0);
final Collection<String> fields = doc.getFieldNames();
assertEquals(3, fields.size());
assertEquals("2020-06-11T11:59:08.386", doc.getField("date_dt").getValue());
assertEquals("other", doc.getField("type_s").getValue());
assertEquals(record, doc.getField("line_t").getValue());
}
private List<SolrInputDocument> readDocs(String records) throws Exception { private List<SolrInputDocument> readDocs(String records) throws Exception {
BufferedReader bufferedReader = new BufferedReader(new StringReader(records)); BufferedReader bufferedReader = new BufferedReader(new StringReader(records));
ArrayList<SolrInputDocument> list = new ArrayList(); ArrayList<SolrInputDocument> list = new ArrayList();