Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-7042: bin/post defaults application/json to /update/json/docs now 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1723879 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erik Hatcher 2016-01-09 19:30:21 +00:00
parent 5e3f72645c
commit 8826ea070a
5 changed files with 25 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
solr/CHANGES.txt
View File

@ -244,6 +244,9 @@ Upgrading from Solr 5.4
* All protected methods from CoreAdminHandler other than handleCustomAction() is removed by SOLR-8476 and can
no more be overridden. If you still wish to override those methods, override the handleRequestBody()
* bin/post now defaults application/json files to the /update/json/docs end-point. Use `-format solr` to force
files to the /update end-point. See SOLR-7042 for more details.
Detailed Change List
----------------------
@ -461,6 +464,9 @@ Other Changes
* SOLR-8505: core/DirectoryFactory.LOCK_TYPE_HDFS - add & use it instead of String literals
(Christine Poerschke)
* SOLR-7042: bin/post now uses /update/json/docs for application/json content types, including support for
.jsonl (JSON Lines) files. (Erik Hatcher and shalin)
================== 5.4.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release

3
solr/bin/post
View File

@ -81,9 +81,10 @@ function print_usage() {
echo " -type <content/type> (default: application/xml)"
echo ""
echo " Other options:"
echo " -filetypes <type>[,<type>,...] (default: xml,json,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log)"
echo " -filetypes <type>[,<type>,...] (default: xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log)"
echo " -params \"<key>=<value>[&<key>=<value>...]\" (values must be URL-encoded; these pass through to Solr update request)"
echo " -out yes|no (default: no; yes outputs Solr response to console)"
echo " -format solr (sends application/json content as Solr commands to /update instead of /update/json/docs)"
echo ""
echo ""
echo "Examples:"

16
solr/core/src/java/org/apache/solr/util/SimplePostTool.java
View File

@ -86,7 +86,7 @@ public class SimplePostTool {
private static final int DEFAULT_WEB_DELAY = 10;
private static final int MAX_WEB_DEPTH = 10;
private static final String DEFAULT_CONTENT_TYPE = "application/xml";
private static final String DEFAULT_FILE_TYPES = "xml,json,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log";
private static final String DEFAULT_FILE_TYPES = "xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log";
static final String DATA_MODE_FILES = "files";
static final String DATA_MODE_ARGS = "args";
@ -102,6 +102,7 @@ public class SimplePostTool {
URL solrUrl;
OutputStream out = null;
String type;
String format;
String mode;
boolean commit;
boolean optimize;
@ -133,6 +134,7 @@ public class SimplePostTool {
mimeMap.put("xml", "application/xml");
mimeMap.put("csv", "text/csv");
mimeMap.put("json", "application/json");
mimeMap.put("jsonl", "application/json");
mimeMap.put("pdf", "application/pdf");
mimeMap.put("rtf", "text/rtf");
mimeMap.put("html", "text/html");
@ -234,6 +236,7 @@ public class SimplePostTool {
URL url = new URL(urlStr);
boolean auto = isOn(System.getProperty("auto", DEFAULT_AUTO));
String type = System.getProperty("type");
String format = System.getProperty("format");
// Recursive
int recursive = 0;
String r = System.getProperty("recursive", DEFAULT_RECURSIVE);
@ -253,7 +256,7 @@ public class SimplePostTool {
boolean commit = isOn(System.getProperty("commit",DEFAULT_COMMIT));
boolean optimize = isOn(System.getProperty("optimize",DEFAULT_OPTIMIZE));
return new SimplePostTool(mode, url, auto, type, recursive, delay, fileTypes, out, commit, optimize, args);
return new SimplePostTool(mode, url, auto, type, format, recursive, delay, fileTypes, out, commit, optimize, args);
} catch (MalformedURLException e) {
fatal("System Property 'url' is not a valid URL: " + urlStr);
return null;
@ -275,13 +278,14 @@ public class SimplePostTool {
* @param optimize if true, will optimize at end of posting
* @param args a String[] of arguments, varies between modes
*/
public SimplePostTool(String mode, URL url, boolean auto, String type,
public SimplePostTool(String mode, URL url, boolean auto, String type, String format,
int recursive, int delay, String fileTypes, OutputStream out,
boolean commit, boolean optimize, String[] args) {
this.mode = mode;
this.solrUrl = url;
this.auto = auto;
this.type = type;
this.format = format;
this.recursive = recursive;
this.delay = delay;
this.fileTypes = fileTypes;
@ -773,7 +777,11 @@ public class SimplePostTool {
}
// TODO: Add a flag that disables /update and sends all to /update/extract, to avoid CSV, JSON, and XML files
// TODO: from being interpreted as Solr documents internally
if(type.equals("application/xml") || type.equals("text/csv") || type.equals("application/json")) {
if (type.equals("application/json") && !"solr".equals(format)) {
suffix = "/json/docs";
String urlStr = appendUrlPath(solrUrl, suffix).toString();
url = new URL(urlStr);
} else if (type.equals("application/xml") || type.equals("text/csv") || type.equals("application/json")) {
// Default handler
} else {
// SolrCell

2
solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java
View File

@ -146,6 +146,8 @@ public class SimplePostToolTest extends SolrTestCaseJ4 {
assertEquals("application/msword", SimplePostTool.guessType(f));
f = new File("foobar");
assertEquals("application/octet-stream", SimplePostTool.guessType(f));
f = new File("foo.jsonl");
assertEquals("application/json", SimplePostTool.guessType(f));
}
@Test

3
solr/example/exampledocs/more_books.jsonl Normal file
View File

@ -0,0 +1,3 @@
{"id":"0060248025","name":"Falling Up","inStock": true,"author": "Shel Silverstein"}
{"id":"0679805273","name":"Oh, The Places You'll Go","inStock": true,"author": "Dr. Seuss"}