Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-14558: Record all log lines in SolrLogPostTool (#1570)

This commit is contained in:
Jason Gerlowski 2020-06-16 08:21:24 -04:00 committed by GitHub
parent a108f90869
commit a7792b129b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 71 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
View File

@ -186,20 +186,25 @@ public class SolrLogPostTool {
}
if (line != null) {
SolrInputDocument lineDoc = new SolrInputDocument();
lineDoc.setField("date_dt", parseDate(line));
lineDoc.setField("line_t", line);
lineDoc.setField("type_s", "other"); // Overridden by known types below
if (line.contains("Registered new searcher")) {
return parseNewSearch(line);
parseNewSearch(lineDoc, line);
} else if (line.contains("path=/update")) {
return parseUpdate(line);
parseUpdate(lineDoc, line);
} else if (line.contains(" ERROR ")) {
this.cause = null;
return parseError(line, readTrace());
parseError(lineDoc, line, readTrace());
} else if (line.contains("start commit")) {
return parseCommit(line);
parseCommit(lineDoc, line);
} else if(line.contains("QTime=")) {
return parseQueryRecord(line);
} else {
continue;
parseQueryRecord(lineDoc, line);
}
return lineDoc;
} else {
return null;
}
@ -252,94 +257,73 @@ public class SolrLogPostTool {
doc.setField(fieldName, fieldValue);
}
private SolrInputDocument parseError(String line, String trace) throws IOException {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));
doc.setField("type_s", "error");
doc.setField("line_t", line);
private void parseError(SolrInputDocument lineRecord, String line, String trace) throws IOException {
lineRecord.setField("type_s", "error");
//Don't include traces that have only the %html header.
if(trace != null && trace.length() > 6) {
doc.setField("stack_t", trace);
lineRecord.setField("stack_t", trace);
}
if(this.cause != null) {
doc.setField("root_cause_t", cause.replace("Caused by:", "").trim());
lineRecord.setField("root_cause_t", cause.replace("Caused by:", "").trim());
}
doc.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line));
doc.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line));
return doc;
lineRecord.setField("collection_s", parseCollection(line));
lineRecord.setField("core_s", parseCore(line));
lineRecord.setField("shard_s", parseShard(line));
lineRecord.setField("replica_s", parseReplica(line));
}
private SolrInputDocument parseCommit(String line) throws IOException {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));
doc.setField("type_s", "commit");
doc.setField("line_t", line);
doc.setField("soft_commit_s", Boolean.toString(line.contains("softCommit=true")));
private void parseCommit(SolrInputDocument lineRecord, String line) throws IOException {
lineRecord.setField("type_s", "commit");
lineRecord.setField("soft_commit_s", Boolean.toString(line.contains("softCommit=true")));
doc.setField("open_searcher_s", Boolean.toString(line.contains("openSearcher=true")));
lineRecord.setField("open_searcher_s", Boolean.toString(line.contains("openSearcher=true")));
doc.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line));
doc.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line));
return doc;
lineRecord.setField("collection_s", parseCollection(line));
lineRecord.setField("core_s", parseCore(line));
lineRecord.setField("shard_s", parseShard(line));
lineRecord.setField("replica_s", parseReplica(line));
}
private SolrInputDocument parseQueryRecord(String line) {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));
doc.setField("qtime_i", parseQTime(line));
doc.setField("status_s", parseStatus(line));
private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
lineRecord.setField("qtime_i", parseQTime(line));
lineRecord.setField("status_s", parseStatus(line));
String path = parsePath(line);
doc.setField("path_s", path);
lineRecord.setField("path_s", path);
if(line.contains("hits=")) {
doc.setField("hits_l", parseHits(line));
lineRecord.setField("hits_l", parseHits(line));
}
String params = parseParams(line);
doc.setField("params_t", params);
addParams(doc, params);
lineRecord.setField("params_t", params);
addParams(lineRecord, params);
doc.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line));
doc.setField("node_s", parseNode(line));
doc.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line));
lineRecord.setField("collection_s", parseCollection(line));
lineRecord.setField("core_s", parseCore(line));
lineRecord.setField("node_s", parseNode(line));
lineRecord.setField("shard_s", parseShard(line));
lineRecord.setField("replica_s", parseReplica(line));
if(path != null && path.contains("/admin")) {
doc.setField("type_s", "admin");
lineRecord.setField("type_s", "admin");
} else if(path != null && params.contains("/replication")) {
doc.setField("type_s", "replication");
lineRecord.setField("type_s", "replication");
} else if (path != null && path.contains("/get")) {
doc.setField("type_s", "get");
lineRecord.setField("type_s", "get");
} else {
doc.setField("type_s", "query");
lineRecord.setField("type_s", "query");
}
return doc;
}
private SolrInputDocument parseNewSearch(String line) {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));
doc.setField("core_s", parseNewSearcherCore(line));
doc.setField("type_s", "newSearcher");
doc.setField("line_t", line);
return doc;
private void parseNewSearch(SolrInputDocument lineRecord, String line) {
lineRecord.setField("core_s", parseNewSearcherCore(line));
lineRecord.setField("type_s", "newSearcher");
}
private String parseCollection(String line) {
@ -352,25 +336,19 @@ public class SolrLogPostTool {
}
}
private SolrInputDocument parseUpdate(String line) {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));
private void parseUpdate(SolrInputDocument lineRecord, String line) {
if(line.contains("deleteByQuery=")) {
doc.setField("type_s", "deleteByQuery");
lineRecord.setField("type_s", "deleteByQuery");
} else if(line.contains("delete=")) {
doc.setField("type_s", "delete");
lineRecord.setField("type_s", "delete");
} else {
doc.setField("type_s", "update");
lineRecord.setField("type_s", "update");
}
doc.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line));
doc.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line));
doc.setField("line_t", line);
return doc;
lineRecord.setField("collection_s", parseCollection(line));
lineRecord.setField("core_s", parseCore(line));
lineRecord.setField("shard_s", parseShard(line));
lineRecord.setField("replica_s", parseReplica(line));
}
private String parseNewSearcherCore(String line) {

16
solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
View File

@ -18,6 +18,7 @@ package org.apache.solr.util;
import java.io.BufferedReader;
import java.io.StringReader;
import java.util.Collection;
import java.util.List;
import java.util.ArrayList;
@ -286,6 +287,21 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
assertEquals(core.getValue(), "production_cv_month_201912_shard35_replica_n1");
}
// Ensure SolrLogPostTool parses _all_ log lines into searchable records
@Test
public void testOtherRecord() throws Exception {
final String record = "2020-06-11 11:59:08.386 INFO (main) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)";
final List<SolrInputDocument> docs = readDocs(record);
assertEquals(docs.size(), 1);
SolrInputDocument doc = docs.get(0);
final Collection<String> fields = doc.getFieldNames();
assertEquals(3, fields.size());
assertEquals("2020-06-11T11:59:08.386", doc.getField("date_dt").getValue());
assertEquals("other", doc.getField("type_s").getValue());
assertEquals(record, doc.getField("line_t").getValue());
}
private List<SolrInputDocument> readDocs(String records) throws Exception {
BufferedReader bufferedReader = new BufferedReader(new StringReader(records));
ArrayList<SolrInputDocument> list = new ArrayList();