From 02bc22d21a303f9324822e39008a3be06e78534b Mon Sep 17 00:00:00 2001 From: Yonik Seeley Date: Wed, 11 May 2011 20:02:54 +0000 Subject: [PATCH] SOLR-2496: add array-of-object JSON format and commitWithin, overwrite request params git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1102058 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 6 + solr/example/exampledocs/books.json | 15 +- .../org/apache/solr/handler/JsonLoader.java | 219 +++++++++++------- .../handler/JsonUpdateRequestHandler.java | 2 +- .../apache/solr/handler/JsonLoaderTest.java | 71 +++++- 5 files changed, 207 insertions(+), 106 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 877183cba80..42555a66b8a 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -270,6 +270,12 @@ Detailed Change List New Features ---------------------- +* SOLR-2496: Add ability to specify overwrite and commitWithin as request + parameters (e.g. specified in the URL) when using the JSON update format, + and added a simplified format for specifying multiple documents. + Example: [{"id":"doc1"},{"id":"doc2"}] + (yonik) + Optimizations ---------------------- diff --git a/solr/example/exampledocs/books.json b/solr/example/exampledocs/books.json index f805c076558..188e0734fbe 100644 --- a/solr/example/exampledocs/books.json +++ b/solr/example/exampledocs/books.json @@ -1,7 +1,5 @@ -{ - -"add": { - "doc": { +[ + { "id" : "978-0641723445", "cat" : ["book","hardcover"], "title" : "The Lightning Thief", @@ -13,11 +11,8 @@ "price" : 12.50, "pages_i" : 384 } -} - , -"add": { - "doc": { + { "id" : "978-1423103349", "cat" : ["book","paperback"], "title" : "The Sea of Monsters", @@ -29,6 +24,4 @@ "price" : 6.49, "pages_i" : 304 } -} - -} +] diff --git a/solr/src/java/org/apache/solr/handler/JsonLoader.java b/solr/src/java/org/apache/solr/handler/JsonLoader.java index c233ce634e4..34118a07402 100644 --- a/solr/src/java/org/apache/solr/handler/JsonLoader.java +++ b/solr/src/java/org/apache/solr/handler/JsonLoader.java @@ -23,6 +23,7 @@ import java.util.Stack; import org.apache.commons.io.IOUtils; import org.apache.noggit.JSONParser; +import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.apache.solr.common.util.ContentStream; @@ -43,10 +44,18 @@ import org.slf4j.LoggerFactory; class JsonLoader extends ContentStreamLoader { final static Logger log = LoggerFactory.getLogger( JsonLoader.class ); - protected UpdateRequestProcessor processor; + protected final UpdateRequestProcessor processor; + protected final SolrQueryRequest req; + protected JSONParser parser; + protected final int commitWithin; + protected final boolean overwrite; - public JsonLoader(UpdateRequestProcessor processor) { + public JsonLoader(SolrQueryRequest req, UpdateRequestProcessor processor) { this.processor = processor; + this.req = req; + + commitWithin = req.getParams().getInt(XmlUpdateRequestHandler.COMMIT_WITHIN, -1); + overwrite = req.getParams().getBool(XmlUpdateRequestHandler.OVERWRITE, true); } @Override @@ -55,14 +64,14 @@ class JsonLoader extends ContentStreamLoader { Reader reader = null; try { reader = stream.getReader(); - if (XmlUpdateRequestHandler.log.isTraceEnabled()) { + if (log.isTraceEnabled()) { String body = IOUtils.toString(reader); - XmlUpdateRequestHandler.log.trace("body", body); + log.trace("body", body); reader = new StringReader(body); } - JSONParser parser = new JSONParser(reader); - this.processUpdate(req, processor, parser); + parser = new JSONParser(reader); + this.processUpdate(); } finally { IOUtils.closeQuietly(reader); @@ -70,39 +79,50 @@ class JsonLoader extends ContentStreamLoader { } @SuppressWarnings("fallthrough") - void processUpdate(SolrQueryRequest req, UpdateRequestProcessor processor, JSONParser parser) throws IOException + void processUpdate() throws IOException { int ev = parser.nextEvent(); while( ev != JSONParser.EOF ) { switch( ev ) { + case JSONParser.ARRAY_START: + handleAdds(); + break; + case JSONParser.STRING: if( parser.wasKey() ) { String v = parser.getString(); if( v.equals( XmlUpdateRequestHandler.ADD ) ) { - processor.processAdd( parseAdd(req, parser ) ); + int ev2 = parser.nextEvent(); + if (ev2 == JSONParser.OBJECT_START) { + processor.processAdd( parseAdd() ); + } else if (ev2 == JSONParser.ARRAY_START) { + handleAdds(); + } else { + assertEvent(ev2, JSONParser.OBJECT_START); + } } else if( v.equals( XmlUpdateRequestHandler.COMMIT ) ) { CommitUpdateCommand cmd = new CommitUpdateCommand(req, false ); cmd.waitFlush = cmd.waitSearcher = true; - parseCommitOptions( parser, cmd ); + parseCommitOptions( cmd ); processor.processCommit( cmd ); } else if( v.equals( XmlUpdateRequestHandler.OPTIMIZE ) ) { CommitUpdateCommand cmd = new CommitUpdateCommand(req, true ); cmd.waitFlush = cmd.waitSearcher = true; - parseCommitOptions( parser, cmd ); + parseCommitOptions( cmd ); processor.processCommit( cmd ); } else if( v.equals( XmlUpdateRequestHandler.DELETE ) ) { - processor.processDelete( parseDelete(req, parser ) ); + processor.processDelete( parseDelete() ); } else if( v.equals( XmlUpdateRequestHandler.ROLLBACK ) ) { - processor.processRollback( parseRollback(req, parser ) ); + processor.processRollback( parseRollback() ); } else { - throw new IOException( "Unknown command: "+v+" ["+parser.getPosition()+"]" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown command: "+v+" ["+parser.getPosition()+"]" ); } break; } @@ -117,12 +137,11 @@ class JsonLoader extends ContentStreamLoader { case JSONParser.OBJECT_START: case JSONParser.OBJECT_END: - case JSONParser.ARRAY_START: case JSONParser.ARRAY_END: break; default: - System.out.println("UNKNOWN_EVENT_ID:"+ev); + log.info("Noggit UNKNOWN_EVENT_ID:"+ev); break; } // read the next event @@ -130,187 +149,211 @@ class JsonLoader extends ContentStreamLoader { } } - DeleteUpdateCommand parseDelete(SolrQueryRequest req, JSONParser js) throws IOException { - assertNextEvent( js, JSONParser.OBJECT_START ); + DeleteUpdateCommand parseDelete() throws IOException { + assertNextEvent( JSONParser.OBJECT_START ); DeleteUpdateCommand cmd = new DeleteUpdateCommand(req); - + while( true ) { - int ev = js.nextEvent(); + int ev = parser.nextEvent(); if( ev == JSONParser.STRING ) { - String key = js.getString(); - if( js.wasKey() ) { + String key = parser.getString(); + if( parser.wasKey() ) { if( "id".equals( key ) ) { - cmd.id = js.getString(); + cmd.id = parser.getString(); } else if( "query".equals(key) ) { - cmd.query = js.getString(); + cmd.query = parser.getString(); } else { - throw new IOException( "Unknown key: "+key+" ["+js.getPosition()+"]" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown key: "+key+" ["+parser.getPosition()+"]" ); } } else { - throw new IOException( + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "invalid string: " + key - +" at ["+js.getPosition()+"]" ); + +" at ["+parser.getPosition()+"]" ); } } else if( ev == JSONParser.OBJECT_END ) { if( cmd.id == null && cmd.query == null ) { - throw new IOException( "Missing id or query for delete ["+js.getPosition()+"]" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Missing id or query for delete ["+parser.getPosition()+"]" ); } return cmd; } else { - throw new IOException( + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Got: "+JSONParser.getEventString( ev ) - +" at ["+js.getPosition()+"]" ); + +" at ["+parser.getPosition()+"]" ); } } } - RollbackUpdateCommand parseRollback(SolrQueryRequest req, JSONParser js) throws IOException { - assertNextEvent( js, JSONParser.OBJECT_START ); - assertNextEvent( js, JSONParser.OBJECT_END ); + RollbackUpdateCommand parseRollback() throws IOException { + assertNextEvent( JSONParser.OBJECT_START ); + assertNextEvent( JSONParser.OBJECT_END ); return new RollbackUpdateCommand(req); } - void parseCommitOptions( JSONParser js, CommitUpdateCommand cmd ) throws IOException + void parseCommitOptions(CommitUpdateCommand cmd ) throws IOException { - assertNextEvent( js, JSONParser.OBJECT_START ); + assertNextEvent( JSONParser.OBJECT_START ); while( true ) { - int ev = js.nextEvent(); + int ev = parser.nextEvent(); if( ev == JSONParser.STRING ) { - String key = js.getString(); - if( js.wasKey() ) { + String key = parser.getString(); + if( parser.wasKey() ) { if( XmlUpdateRequestHandler.WAIT_SEARCHER.equals( key ) ) { - cmd.waitSearcher = js.getBoolean(); + cmd.waitSearcher = parser.getBoolean(); } else if( XmlUpdateRequestHandler.WAIT_FLUSH.equals( key ) ) { - cmd.waitFlush = js.getBoolean(); + cmd.waitFlush = parser.getBoolean(); } else { - throw new IOException( "Unknown key: "+key+" ["+js.getPosition()+"]" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown key: "+key+" ["+parser.getPosition()+"]" ); } } else { - throw new IOException( + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "invalid string: " + key - +" at ["+js.getPosition()+"]" ); + +" at ["+parser.getPosition()+"]" ); } } else if( ev == JSONParser.OBJECT_END ) { return; } else { - throw new IOException( + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Got: "+JSONParser.getEventString( ev ) - +" at ["+js.getPosition()+"]" ); + +" at ["+parser.getPosition()+"]" ); } } } - AddUpdateCommand parseAdd(SolrQueryRequest req, JSONParser js ) throws IOException + AddUpdateCommand parseAdd() throws IOException { - assertNextEvent( js, JSONParser.OBJECT_START ); AddUpdateCommand cmd = new AddUpdateCommand(req); + cmd.commitWithin = commitWithin; + cmd.overwrite = overwrite; + float boost = 1.0f; while( true ) { - int ev = js.nextEvent(); + int ev = parser.nextEvent(); if( ev == JSONParser.STRING ) { - if( js.wasKey() ) { - String key = js.getString(); + if( parser.wasKey() ) { + String key = parser.getString(); if( "doc".equals( key ) ) { if( cmd.solrDoc != null ) { - throw new IOException( "multiple docs in same add command" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "multiple docs in same add command" ); } - ev = assertNextEvent( js, JSONParser.OBJECT_START ); - cmd.solrDoc = parseDoc( ev, js ); + ev = assertNextEvent( JSONParser.OBJECT_START ); + cmd.solrDoc = parseDoc( ev ); } else if( XmlUpdateRequestHandler.OVERWRITE.equals( key ) ) { - cmd.overwrite = js.getBoolean(); // reads next boolean + cmd.overwrite = parser.getBoolean(); // reads next boolean } else if( XmlUpdateRequestHandler.COMMIT_WITHIN.equals( key ) ) { - cmd.commitWithin = (int)js.getLong(); + cmd.commitWithin = (int)parser.getLong(); } else if( "boost".equals( key ) ) { - boost = Float.parseFloat( js.getNumberChars().toString() ); + boost = Float.parseFloat( parser.getNumberChars().toString() ); } else { - throw new IOException( "Unknown key: "+key+" ["+js.getPosition()+"]" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown key: "+key+" ["+parser.getPosition()+"]" ); } } else { - throw new IOException( + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Should be a key " - +" at ["+js.getPosition()+"]" ); + +" at ["+parser.getPosition()+"]" ); } } else if( ev == JSONParser.OBJECT_END ) { if( cmd.solrDoc == null ) { - throw new IOException("missing solr document. "+js.getPosition() ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"missing solr document. "+parser.getPosition() ); } cmd.solrDoc.setDocumentBoost( boost ); return cmd; } else { - throw new IOException( + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Got: "+JSONParser.getEventString( ev ) - +" at ["+js.getPosition()+"]" ); + +" at ["+parser.getPosition()+"]" ); } } } - - int assertNextEvent( JSONParser parser, int ev ) throws IOException + + + void handleAdds() throws IOException + { + while( true ) { + AddUpdateCommand cmd = new AddUpdateCommand(req); + cmd.commitWithin = commitWithin; + cmd.overwrite = overwrite; + + int ev = parser.nextEvent(); + if (ev == JSONParser.ARRAY_END) break; + + assertEvent(ev, JSONParser.OBJECT_START); + cmd.solrDoc = parseDoc(ev); + processor.processAdd(cmd); + } + } + + + int assertNextEvent(int expected ) throws IOException { int got = parser.nextEvent(); - if( ev != got ) { - throw new IOException( - "Expected: "+JSONParser.getEventString( ev ) - +" but got "+JSONParser.getEventString( got ) - +" at ["+parser.getPosition()+"]" ); - } + assertEvent(got, expected); return got; } + + void assertEvent(int ev, int expected) { + if( ev != expected ) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Expected: "+JSONParser.getEventString( expected ) + +" but got "+JSONParser.getEventString( ev ) + +" at ["+parser.getPosition()+"]" ); + } + } - SolrInputDocument parseDoc( int ev, JSONParser js ) throws IOException + SolrInputDocument parseDoc(int ev) throws IOException { Stack stack = new Stack(); Object obj = null; boolean inArray = false; if( ev != JSONParser.OBJECT_START ) { - throw new IOException( "object should already be started" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "object should already be started" ); } while( true ) { - //System.out.println( ev + "["+JSONParser.getEventString(ev)+"] "+js.wasKey() ); //+ js.getString() ); + //System.out.println( ev + "["+JSONParser.getEventString(ev)+"] "+parser.wasKey() ); //+ parser.getString() ); switch (ev) { case JSONParser.STRING: - if( js.wasKey() ) { + if( parser.wasKey() ) { obj = stack.peek(); - String v = js.getString(); + String v = parser.getString(); if( obj instanceof SolrInputField ) { SolrInputField field = (SolrInputField)obj; if( "boost".equals( v ) ) { - ev = js.nextEvent(); + ev = parser.nextEvent(); if( ev != JSONParser.NUMBER && ev != JSONParser.LONG && ev != JSONParser.BIGNUMBER ) { - throw new IOException( "boost should have number! "+JSONParser.getEventString(ev) ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "boost should have number! "+JSONParser.getEventString(ev) ); } - field.setBoost( Float.valueOf( js.getNumberChars().toString() ) ); + field.setBoost( Float.valueOf( parser.getNumberChars().toString() ) ); } else if( "value".equals( v ) ) { // nothing special... stack.push( field ); // so it can be popped } else { - throw new IOException( "invalid key: "+v + " ["+js.getPosition()+"]" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "invalid key: "+v + " ["+ parser.getPosition()+"]" ); } } else if( obj instanceof SolrInputDocument ) { @@ -323,22 +366,22 @@ class JsonLoader extends ContentStreamLoader { stack.push( f ); } else { - throw new IOException( "hymmm ["+js.getPosition()+"]" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "hymmm ["+ parser.getPosition()+"]" ); } } else { - addValToField(stack, js.getString(), inArray, js); + addValToField(stack, parser.getString(), inArray, parser); } break; case JSONParser.LONG: case JSONParser.NUMBER: case JSONParser.BIGNUMBER: - addValToField(stack, js.getNumberChars().toString(), inArray, js); + addValToField(stack, parser.getNumberChars().toString(), inArray, parser); break; case JSONParser.BOOLEAN: - addValToField(stack, js.getBoolean(),inArray, js); + addValToField(stack, parser.getBoolean(),inArray, parser); break; case JSONParser.OBJECT_START: @@ -351,7 +394,7 @@ class JsonLoader extends ContentStreamLoader { // should alreay be pushed... } else { - throw new IOException( "should not start new object with: "+obj + " ["+js.getPosition()+"]" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "should not start new object with: "+obj + " ["+ parser.getPosition()+"]" ); } } break; @@ -365,7 +408,7 @@ class JsonLoader extends ContentStreamLoader { // should already be pushed... } else { - throw new IOException( "should not start new object with: "+obj + " ["+js.getPosition()+"]" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "should not start new object with: "+obj + " ["+ parser.getPosition()+"]" ); } break; @@ -383,18 +426,18 @@ class JsonLoader extends ContentStreamLoader { break; } - ev = js.nextEvent(); + ev = parser.nextEvent(); if( ev == JSONParser.EOF ) { - throw new IOException( "should finish doc first!" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "should finish doc first!" ); } } } - static void addValToField( Stack stack, Object val, boolean inArray, JSONParser js ) throws IOException + static void addValToField( Stack stack, Object val, boolean inArray, JSONParser parser ) throws IOException { Object obj = stack.peek(); if( !(obj instanceof SolrInputField) ) { - throw new IOException( "hymmm ["+js.getPosition()+"]" ); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "hymmm ["+parser.getPosition()+"]" ); } SolrInputField f = inArray diff --git a/solr/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java b/solr/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java index 9f36c37d785..213089d0943 100644 --- a/solr/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java +++ b/solr/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java @@ -37,7 +37,7 @@ public class JsonUpdateRequestHandler extends ContentStreamHandlerBase { @Override protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) { - return new JsonLoader(processor); + return new JsonLoader(req, processor); } //////////////////////// SolrInfoMBeans methods ////////////////////// diff --git a/solr/src/test/org/apache/solr/handler/JsonLoaderTest.java b/solr/src/test/org/apache/solr/handler/JsonLoaderTest.java index e6635475356..5deec94f01b 100644 --- a/solr/src/test/org/apache/solr/handler/JsonLoaderTest.java +++ b/solr/src/test/org/apache/solr/handler/JsonLoaderTest.java @@ -26,7 +26,9 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; +import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.update.AddUpdateCommand; import org.apache.solr.update.CommitUpdateCommand; import org.apache.solr.update.DeleteUpdateCommand; @@ -81,13 +83,11 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { public void testParsing() throws Exception { SolrQueryRequest req = req(); - Reader reader = new StringReader(input); - + SolrQueryResponse rsp = new SolrQueryResponse(); BufferingRequestProcessor p = new BufferingRequestProcessor(null); - JsonLoader loader = new JsonLoader( p ); - - loader.processUpdate(req, p, new JSONParser(reader) ); - + JsonLoader loader = new JsonLoader( req, p ); + loader.load(req, rsp, new ContentStreamBase.StringStream(input)); + assertEquals( 2, p.addCommands.size() ); AddUpdateCommand add = p.addCommands.get(0); @@ -133,8 +133,67 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { req.close(); } + + + public void testSimpleFormat() throws Exception + { + String str = "[{'id':'1'},{'id':'2'}]".replace('\'', '"'); + SolrQueryRequest req = req("commitWithin","100", "overwrite","false"); + SolrQueryResponse rsp = new SolrQueryResponse(); + BufferingRequestProcessor p = new BufferingRequestProcessor(null); + JsonLoader loader = new JsonLoader( req, p ); + loader.load(req, rsp, new ContentStreamBase.StringStream(str)); + + assertEquals( 2, p.addCommands.size() ); + + AddUpdateCommand add = p.addCommands.get(0); + SolrInputDocument d = add.solrDoc; + SolrInputField f = d.getField( "id" ); + assertEquals("1", f.getValue()); + assertEquals(add.commitWithin, 100); + assertEquals(add.overwrite, false); + + add = p.addCommands.get(1); + d = add.solrDoc; + f = d.getField( "id" ); + assertEquals("2", f.getValue()); + assertEquals(add.commitWithin, 100); + assertEquals(add.overwrite, false); + + req.close(); + } + + public void testSimpleFormatInAdd() throws Exception + { + String str = "{'add':[{'id':'1'},{'id':'2'}]}".replace('\'', '"'); + SolrQueryRequest req = req(); + SolrQueryResponse rsp = new SolrQueryResponse(); + BufferingRequestProcessor p = new BufferingRequestProcessor(null); + JsonLoader loader = new JsonLoader( req, p ); + loader.load(req, rsp, new ContentStreamBase.StringStream(str)); + + assertEquals( 2, p.addCommands.size() ); + + AddUpdateCommand add = p.addCommands.get(0); + SolrInputDocument d = add.solrDoc; + SolrInputField f = d.getField( "id" ); + assertEquals("1", f.getValue()); + assertEquals(add.commitWithin, -1); + assertEquals(add.overwrite, true); + + add = p.addCommands.get(1); + d = add.solrDoc; + f = d.getField( "id" ); + assertEquals("2", f.getValue()); + assertEquals(add.commitWithin, -1); + assertEquals(add.overwrite, true); + + req.close(); + } + } + class BufferingRequestProcessor extends UpdateRequestProcessor { List addCommands = new ArrayList();