Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-14130: Get working with branch_8x

This commit is contained in:
Joel Bernstein 2020-01-03 09:51:18 -05:00
parent f35cd71427
commit 861bdae964
1 changed files with 43 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
View File

@ -39,17 +39,17 @@ public class SolrLogPostTool {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
if(args.length != 2) { if(args.length != 2) {
CLIO.out(""); System.out.println("");
CLIO.out("postlogs is a simple tool for indexing Solr logs."); System.out.println("postlogs is a simple tool for indexing Solr logs.");
CLIO.out(""); System.out.println("");
CLIO.out("parameters:"); System.out.println("parameters:");
CLIO.out(""); System.out.println("");
CLIO.out("-- baseUrl: Example http://localhost:8983/solr/collection1"); System.out.println("-- baseUrl: Example http://localhost:8983/solr/collection1");
CLIO.out("-- rootDir: All files found at or below the root will be indexed."); System.out.println("-- rootDir: All files found at or below the root will be indexed.");
CLIO.out(""); System.out.println("");
CLIO.out("Sample syntax 1: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs/solr.log"); System.out.println("Sample syntax 1: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs/solr.log");
CLIO.out("Sample syntax 2: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs"); System.out.println("Sample syntax 2: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs");
CLIO.out(""); System.out.println("");
return; return;
} }
@ -79,8 +79,8 @@ public class SolrLogPostTool {
try { try {
doc = recordReader.readRecord(); doc = recordReader.readRecord();
} catch (Throwable t) { } catch (Throwable t) {
CLIO.err("Error reading log record:"+ bufferedReader.getLineNumber() +" from file:"+ fileName); System.err.println("Error reading log record:"+ bufferedReader.getLineNumber() +" from file:"+ fileName);
CLIO.err(t.getMessage()); System.err.println(t.getMessage());
continue; continue;
} }
@ -94,9 +94,9 @@ public class SolrLogPostTool {
doc.addField("file_s", fileName); doc.addField("file_s", fileName);
request.add(doc); request.add(doc);
if (rec == 300) { if (rec == 300) {
CLIO.out("Sending batch of 300 log records..."); System.out.println("Sending batch of 300 log records...");
request.process(client); request.process(client);
CLIO.out("Batch sent"); System.out.println("Batch sent");
request = new UpdateRequest(); request = new UpdateRequest();
rec = 0; rec = 0;
} }
@ -108,10 +108,10 @@ public class SolrLogPostTool {
if (rec > 0) { if (rec > 0) {
//Process last batch //Process last batch
CLIO.out("Sending last batch ..."); System.out.println("Sending last batch ...");
request.process(client); request.process(client);
client.commit(); client.commit();
CLIO.out("Committed"); System.out.println("Committed");
} }
} finally { } finally {
client.close(); client.close();
@ -270,7 +270,7 @@ public class SolrLogPostTool {
return doc; return doc;
} }
private SolrInputDocument parseQueryRecord(String line) { private SolrInputDocument parseQueryRecord(String line) throws IOException {
SolrInputDocument doc = new SolrInputDocument(); SolrInputDocument doc = new SolrInputDocument();
doc.addField("date_dt", parseDate(line)); doc.addField("date_dt", parseDate(line));
@ -460,39 +460,43 @@ public class SolrLogPostTool {
return builder.toString(); return builder.toString();
} }
private void addParams(SolrInputDocument doc, String params) { private void addParams(SolrInputDocument doc, String params) throws IOException {
String[] pairs = params.split("&"); String[] pairs = params.split("&");
for(String pair : pairs) { for(String pair : pairs) {
String[] parts = pair.split("="); String[] parts = pair.split("=");
if(parts.length == 2 && parts[0].equals("q")) { if(parts.length == 2 && parts[0].equals("q")) {
String dq = URLDecoder.decode(parts[1], Charset.defaultCharset()); String dq = URLDecoder.decode(parts[1], "UTF-8");
doc.addField("q_s", dq); doc.addField("q_s", dq);
doc.addField("q_t", dq); doc.addField("q_t", dq);
} }
if(parts[0].equals("rows")) { try {
String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); if (parts[0].equals("rows")) {
doc.addField("rows_i", dr); String dr = URLDecoder.decode(parts[1], "UTF-8");
} doc.addField("rows_i", dr);
}
if(parts[0].equals("distrib")) { if (parts[0].equals("distrib")) {
String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); String dr = URLDecoder.decode(parts[1], "UTF-8");
doc.addField("distrib_s", dr); doc.addField("distrib_s", dr);
} }
if(parts[0].equals("isShard")) { if (parts[0].equals("isShard")) {
String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); String dr = URLDecoder.decode(parts[1], "UTF-8");
doc.addField("isShard_s", dr); doc.addField("isShard_s", dr);
} }
if(parts[0].equals("wt")) { if (parts[0].equals("wt")) {
String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); String dr = URLDecoder.decode(parts[1], "UTF-8");
doc.addField("wt_s", dr); doc.addField("wt_s", dr);
} }
if(parts[0].equals("facet")) { if (parts[0].equals("facet")) {
String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); String dr = URLDecoder.decode(parts[1], "UTF-8");
doc.addField("facet_s", dr); doc.addField("facet_s", dr);
}
} catch (Exception e) {
throw new IOException(e);
} }
} }
} }