mirror of https://github.com/apache/lucene.git
SOLR-14558: Record all log lines in SolrLogPostTool (#1570)
This commit is contained in:
parent
a108f90869
commit
a7792b129b
|
@ -186,20 +186,25 @@ public class SolrLogPostTool {
|
|||
}
|
||||
|
||||
if (line != null) {
|
||||
SolrInputDocument lineDoc = new SolrInputDocument();
|
||||
lineDoc.setField("date_dt", parseDate(line));
|
||||
lineDoc.setField("line_t", line);
|
||||
lineDoc.setField("type_s", "other"); // Overridden by known types below
|
||||
|
||||
if (line.contains("Registered new searcher")) {
|
||||
return parseNewSearch(line);
|
||||
parseNewSearch(lineDoc, line);
|
||||
} else if (line.contains("path=/update")) {
|
||||
return parseUpdate(line);
|
||||
parseUpdate(lineDoc, line);
|
||||
} else if (line.contains(" ERROR ")) {
|
||||
this.cause = null;
|
||||
return parseError(line, readTrace());
|
||||
parseError(lineDoc, line, readTrace());
|
||||
} else if (line.contains("start commit")) {
|
||||
return parseCommit(line);
|
||||
parseCommit(lineDoc, line);
|
||||
} else if(line.contains("QTime=")) {
|
||||
return parseQueryRecord(line);
|
||||
} else {
|
||||
continue;
|
||||
parseQueryRecord(lineDoc, line);
|
||||
}
|
||||
|
||||
return lineDoc;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
@ -252,94 +257,73 @@ public class SolrLogPostTool {
|
|||
doc.setField(fieldName, fieldValue);
|
||||
}
|
||||
|
||||
private SolrInputDocument parseError(String line, String trace) throws IOException {
|
||||
SolrInputDocument doc = new SolrInputDocument();
|
||||
doc.setField("date_dt", parseDate(line));
|
||||
doc.setField("type_s", "error");
|
||||
doc.setField("line_t", line);
|
||||
private void parseError(SolrInputDocument lineRecord, String line, String trace) throws IOException {
|
||||
lineRecord.setField("type_s", "error");
|
||||
|
||||
//Don't include traces that have only the %html header.
|
||||
if(trace != null && trace.length() > 6) {
|
||||
doc.setField("stack_t", trace);
|
||||
lineRecord.setField("stack_t", trace);
|
||||
}
|
||||
|
||||
if(this.cause != null) {
|
||||
doc.setField("root_cause_t", cause.replace("Caused by:", "").trim());
|
||||
lineRecord.setField("root_cause_t", cause.replace("Caused by:", "").trim());
|
||||
}
|
||||
|
||||
doc.setField("collection_s", parseCollection(line));
|
||||
doc.setField("core_s", parseCore(line));
|
||||
doc.setField("shard_s", parseShard(line));
|
||||
doc.setField("replica_s", parseReplica(line));
|
||||
|
||||
return doc;
|
||||
lineRecord.setField("collection_s", parseCollection(line));
|
||||
lineRecord.setField("core_s", parseCore(line));
|
||||
lineRecord.setField("shard_s", parseShard(line));
|
||||
lineRecord.setField("replica_s", parseReplica(line));
|
||||
}
|
||||
|
||||
private SolrInputDocument parseCommit(String line) throws IOException {
|
||||
SolrInputDocument doc = new SolrInputDocument();
|
||||
doc.setField("date_dt", parseDate(line));
|
||||
doc.setField("type_s", "commit");
|
||||
doc.setField("line_t", line);
|
||||
doc.setField("soft_commit_s", Boolean.toString(line.contains("softCommit=true")));
|
||||
private void parseCommit(SolrInputDocument lineRecord, String line) throws IOException {
|
||||
lineRecord.setField("type_s", "commit");
|
||||
lineRecord.setField("soft_commit_s", Boolean.toString(line.contains("softCommit=true")));
|
||||
|
||||
doc.setField("open_searcher_s", Boolean.toString(line.contains("openSearcher=true")));
|
||||
lineRecord.setField("open_searcher_s", Boolean.toString(line.contains("openSearcher=true")));
|
||||
|
||||
doc.setField("collection_s", parseCollection(line));
|
||||
doc.setField("core_s", parseCore(line));
|
||||
doc.setField("shard_s", parseShard(line));
|
||||
doc.setField("replica_s", parseReplica(line));
|
||||
|
||||
return doc;
|
||||
lineRecord.setField("collection_s", parseCollection(line));
|
||||
lineRecord.setField("core_s", parseCore(line));
|
||||
lineRecord.setField("shard_s", parseShard(line));
|
||||
lineRecord.setField("replica_s", parseReplica(line));
|
||||
}
|
||||
|
||||
private SolrInputDocument parseQueryRecord(String line) {
|
||||
|
||||
SolrInputDocument doc = new SolrInputDocument();
|
||||
doc.setField("date_dt", parseDate(line));
|
||||
doc.setField("qtime_i", parseQTime(line));
|
||||
doc.setField("status_s", parseStatus(line));
|
||||
private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
|
||||
lineRecord.setField("qtime_i", parseQTime(line));
|
||||
lineRecord.setField("status_s", parseStatus(line));
|
||||
|
||||
String path = parsePath(line);
|
||||
doc.setField("path_s", path);
|
||||
lineRecord.setField("path_s", path);
|
||||
|
||||
if(line.contains("hits=")) {
|
||||
doc.setField("hits_l", parseHits(line));
|
||||
lineRecord.setField("hits_l", parseHits(line));
|
||||
}
|
||||
|
||||
String params = parseParams(line);
|
||||
doc.setField("params_t", params);
|
||||
addParams(doc, params);
|
||||
lineRecord.setField("params_t", params);
|
||||
addParams(lineRecord, params);
|
||||
|
||||
doc.setField("collection_s", parseCollection(line));
|
||||
doc.setField("core_s", parseCore(line));
|
||||
doc.setField("node_s", parseNode(line));
|
||||
doc.setField("shard_s", parseShard(line));
|
||||
doc.setField("replica_s", parseReplica(line));
|
||||
lineRecord.setField("collection_s", parseCollection(line));
|
||||
lineRecord.setField("core_s", parseCore(line));
|
||||
lineRecord.setField("node_s", parseNode(line));
|
||||
lineRecord.setField("shard_s", parseShard(line));
|
||||
lineRecord.setField("replica_s", parseReplica(line));
|
||||
|
||||
|
||||
if(path != null && path.contains("/admin")) {
|
||||
doc.setField("type_s", "admin");
|
||||
lineRecord.setField("type_s", "admin");
|
||||
} else if(path != null && params.contains("/replication")) {
|
||||
doc.setField("type_s", "replication");
|
||||
lineRecord.setField("type_s", "replication");
|
||||
} else if (path != null && path.contains("/get")) {
|
||||
doc.setField("type_s", "get");
|
||||
lineRecord.setField("type_s", "get");
|
||||
} else {
|
||||
doc.setField("type_s", "query");
|
||||
lineRecord.setField("type_s", "query");
|
||||
}
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
|
||||
private SolrInputDocument parseNewSearch(String line) {
|
||||
|
||||
SolrInputDocument doc = new SolrInputDocument();
|
||||
doc.setField("date_dt", parseDate(line));
|
||||
doc.setField("core_s", parseNewSearcherCore(line));
|
||||
doc.setField("type_s", "newSearcher");
|
||||
doc.setField("line_t", line);
|
||||
|
||||
return doc;
|
||||
private void parseNewSearch(SolrInputDocument lineRecord, String line) {
|
||||
lineRecord.setField("core_s", parseNewSearcherCore(line));
|
||||
lineRecord.setField("type_s", "newSearcher");
|
||||
}
|
||||
|
||||
private String parseCollection(String line) {
|
||||
|
@ -352,25 +336,19 @@ public class SolrLogPostTool {
|
|||
}
|
||||
}
|
||||
|
||||
private SolrInputDocument parseUpdate(String line) {
|
||||
SolrInputDocument doc = new SolrInputDocument();
|
||||
doc.setField("date_dt", parseDate(line));
|
||||
|
||||
private void parseUpdate(SolrInputDocument lineRecord, String line) {
|
||||
if(line.contains("deleteByQuery=")) {
|
||||
doc.setField("type_s", "deleteByQuery");
|
||||
lineRecord.setField("type_s", "deleteByQuery");
|
||||
} else if(line.contains("delete=")) {
|
||||
doc.setField("type_s", "delete");
|
||||
lineRecord.setField("type_s", "delete");
|
||||
} else {
|
||||
doc.setField("type_s", "update");
|
||||
lineRecord.setField("type_s", "update");
|
||||
}
|
||||
|
||||
doc.setField("collection_s", parseCollection(line));
|
||||
doc.setField("core_s", parseCore(line));
|
||||
doc.setField("shard_s", parseShard(line));
|
||||
doc.setField("replica_s", parseReplica(line));
|
||||
doc.setField("line_t", line);
|
||||
|
||||
return doc;
|
||||
lineRecord.setField("collection_s", parseCollection(line));
|
||||
lineRecord.setField("core_s", parseCore(line));
|
||||
lineRecord.setField("shard_s", parseShard(line));
|
||||
lineRecord.setField("replica_s", parseReplica(line));
|
||||
}
|
||||
|
||||
private String parseNewSearcherCore(String line) {
|
||||
|
|
|
@ -18,6 +18,7 @@ package org.apache.solr.util;
|
|||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.StringReader;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
|
@ -286,6 +287,21 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
|
|||
assertEquals(core.getValue(), "production_cv_month_201912_shard35_replica_n1");
|
||||
}
|
||||
|
||||
// Ensure SolrLogPostTool parses _all_ log lines into searchable records
|
||||
@Test
|
||||
public void testOtherRecord() throws Exception {
|
||||
final String record = "2020-06-11 11:59:08.386 INFO (main) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)";
|
||||
final List<SolrInputDocument> docs = readDocs(record);
|
||||
assertEquals(docs.size(), 1);
|
||||
|
||||
SolrInputDocument doc = docs.get(0);
|
||||
final Collection<String> fields = doc.getFieldNames();
|
||||
assertEquals(3, fields.size());
|
||||
assertEquals("2020-06-11T11:59:08.386", doc.getField("date_dt").getValue());
|
||||
assertEquals("other", doc.getField("type_s").getValue());
|
||||
assertEquals(record, doc.getField("line_t").getValue());
|
||||
}
|
||||
|
||||
private List<SolrInputDocument> readDocs(String records) throws Exception {
|
||||
BufferedReader bufferedReader = new BufferedReader(new StringReader(records));
|
||||
ArrayList<SolrInputDocument> list = new ArrayList();
|
||||
|
|
Loading…
Reference in New Issue