From 8826ea070ae7a201334f50d4d298ea7303921b66 Mon Sep 17 00:00:00 2001 From: Erik Hatcher Date: Sat, 9 Jan 2016 19:30:21 +0000 Subject: [PATCH] SOLR-7042: bin/post defaults application/json to /update/json/docs now git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1723879 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 6 ++++++ solr/bin/post | 3 ++- .../org/apache/solr/util/SimplePostTool.java | 16 ++++++++++++---- .../org/apache/solr/util/SimplePostToolTest.java | 2 ++ solr/example/exampledocs/more_books.jsonl | 3 +++ 5 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 solr/example/exampledocs/more_books.jsonl diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d1ce0ddfd19..8810449c9a4 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -244,6 +244,9 @@ Upgrading from Solr 5.4 * All protected methods from CoreAdminHandler other than handleCustomAction() is removed by SOLR-8476 and can no more be overridden. If you still wish to override those methods, override the handleRequestBody() +* bin/post now defaults application/json files to the /update/json/docs end-point. Use `-format solr` to force + files to the /update end-point. See SOLR-7042 for more details. + Detailed Change List ---------------------- @@ -461,6 +464,9 @@ Other Changes * SOLR-8505: core/DirectoryFactory.LOCK_TYPE_HDFS - add & use it instead of String literals (Christine Poerschke) +* SOLR-7042: bin/post now uses /update/json/docs for application/json content types, including support for + .jsonl (JSON Lines) files. (Erik Hatcher and shalin) + ================== 5.4.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release diff --git a/solr/bin/post b/solr/bin/post index 39b2431c656..92172ab2429 100755 --- a/solr/bin/post +++ b/solr/bin/post @@ -81,9 +81,10 @@ function print_usage() { echo " -type (default: application/xml)" echo "" echo " Other options:" - echo " -filetypes [,,...] (default: xml,json,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log)" + echo " -filetypes [,,...] (default: xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log)" echo " -params \"=[&=...]\" (values must be URL-encoded; these pass through to Solr update request)" echo " -out yes|no (default: no; yes outputs Solr response to console)" + echo " -format solr (sends application/json content as Solr commands to /update instead of /update/json/docs)" echo "" echo "" echo "Examples:" diff --git a/solr/core/src/java/org/apache/solr/util/SimplePostTool.java b/solr/core/src/java/org/apache/solr/util/SimplePostTool.java index 2947bfdf859..f12709eee98 100644 --- a/solr/core/src/java/org/apache/solr/util/SimplePostTool.java +++ b/solr/core/src/java/org/apache/solr/util/SimplePostTool.java @@ -86,7 +86,7 @@ public class SimplePostTool { private static final int DEFAULT_WEB_DELAY = 10; private static final int MAX_WEB_DEPTH = 10; private static final String DEFAULT_CONTENT_TYPE = "application/xml"; - private static final String DEFAULT_FILE_TYPES = "xml,json,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log"; + private static final String DEFAULT_FILE_TYPES = "xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log"; static final String DATA_MODE_FILES = "files"; static final String DATA_MODE_ARGS = "args"; @@ -102,6 +102,7 @@ public class SimplePostTool { URL solrUrl; OutputStream out = null; String type; + String format; String mode; boolean commit; boolean optimize; @@ -133,6 +134,7 @@ public class SimplePostTool { mimeMap.put("xml", "application/xml"); mimeMap.put("csv", "text/csv"); mimeMap.put("json", "application/json"); + mimeMap.put("jsonl", "application/json"); mimeMap.put("pdf", "application/pdf"); mimeMap.put("rtf", "text/rtf"); mimeMap.put("html", "text/html"); @@ -234,6 +236,7 @@ public class SimplePostTool { URL url = new URL(urlStr); boolean auto = isOn(System.getProperty("auto", DEFAULT_AUTO)); String type = System.getProperty("type"); + String format = System.getProperty("format"); // Recursive int recursive = 0; String r = System.getProperty("recursive", DEFAULT_RECURSIVE); @@ -253,7 +256,7 @@ public class SimplePostTool { boolean commit = isOn(System.getProperty("commit",DEFAULT_COMMIT)); boolean optimize = isOn(System.getProperty("optimize",DEFAULT_OPTIMIZE)); - return new SimplePostTool(mode, url, auto, type, recursive, delay, fileTypes, out, commit, optimize, args); + return new SimplePostTool(mode, url, auto, type, format, recursive, delay, fileTypes, out, commit, optimize, args); } catch (MalformedURLException e) { fatal("System Property 'url' is not a valid URL: " + urlStr); return null; @@ -275,13 +278,14 @@ public class SimplePostTool { * @param optimize if true, will optimize at end of posting * @param args a String[] of arguments, varies between modes */ - public SimplePostTool(String mode, URL url, boolean auto, String type, + public SimplePostTool(String mode, URL url, boolean auto, String type, String format, int recursive, int delay, String fileTypes, OutputStream out, boolean commit, boolean optimize, String[] args) { this.mode = mode; this.solrUrl = url; this.auto = auto; this.type = type; + this.format = format; this.recursive = recursive; this.delay = delay; this.fileTypes = fileTypes; @@ -773,7 +777,11 @@ public class SimplePostTool { } // TODO: Add a flag that disables /update and sends all to /update/extract, to avoid CSV, JSON, and XML files // TODO: from being interpreted as Solr documents internally - if(type.equals("application/xml") || type.equals("text/csv") || type.equals("application/json")) { + if (type.equals("application/json") && !"solr".equals(format)) { + suffix = "/json/docs"; + String urlStr = appendUrlPath(solrUrl, suffix).toString(); + url = new URL(urlStr); + } else if (type.equals("application/xml") || type.equals("text/csv") || type.equals("application/json")) { // Default handler } else { // SolrCell diff --git a/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java b/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java index e8fae95a0d4..29576ad549a 100644 --- a/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java +++ b/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java @@ -146,6 +146,8 @@ public class SimplePostToolTest extends SolrTestCaseJ4 { assertEquals("application/msword", SimplePostTool.guessType(f)); f = new File("foobar"); assertEquals("application/octet-stream", SimplePostTool.guessType(f)); + f = new File("foo.jsonl"); + assertEquals("application/json", SimplePostTool.guessType(f)); } @Test diff --git a/solr/example/exampledocs/more_books.jsonl b/solr/example/exampledocs/more_books.jsonl new file mode 100644 index 00000000000..a48ad1e423a --- /dev/null +++ b/solr/example/exampledocs/more_books.jsonl @@ -0,0 +1,3 @@ +{"id":"0060248025","name":"Falling Up","inStock": true,"author": "Shel Silverstein"} +{"id":"0679805273","name":"Oh, The Places You'll Go","inStock": true,"author": "Dr. Seuss"} +