diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 7e305ef115e..78354175e84 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -162,6 +162,8 @@ Bug Fixes * SOLR-5260: Facet search on a docvalue field in a multi shard collection (Trym Møller, Erick Erickson) +* SOLR-9768: RecordingJsonParser produces incomplete json (Wojciech Stryszyk via ab) + Other Changes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java b/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java index 030913d349f..a85610b7cb2 100644 --- a/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java +++ b/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java @@ -29,7 +29,9 @@ public class RecordingJSONParser extends JSONParser { private StringBuilder sb = new StringBuilder(); private boolean objectStarted = false; - public long lastMarkedPosition = 0; + private long lastMarkedPosition = 0; + private long lastGlobalPosition = 0; + private static final int BUFFER_SIZE = 8192; public RecordingJSONParser(Reader in) { @@ -39,7 +41,7 @@ public class RecordingJSONParser extends JSONParser { } static char[] getChars() { - buf.set(new char[8192]); + buf.set(new char[BUFFER_SIZE]); return buf.get(); } @@ -68,11 +70,22 @@ public class RecordingJSONParser extends JSONParser { if(currPosition < 0){ System.out.println("ERROR"); } + if (currPosition > lastMarkedPosition) { for (long i = lastMarkedPosition; i < currPosition; i++) { recordChar(bufCopy[(int) i]); } + } else if (currPosition < lastMarkedPosition) { + for (long i = 0; i < currPosition; i++) { + recordChar(bufCopy[(int) i]); + } + } else if (currPosition == BUFFER_SIZE && lastGlobalPosition != globalPosition) { + for (long i = 0; i < currPosition; i++) { + recordChar(bufCopy[(int) i]); + } } + + lastGlobalPosition = globalPosition; lastMarkedPosition = currPosition; } diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java b/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java index d59dea39146..da75a43cf15 100644 --- a/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java @@ -16,11 +16,6 @@ */ package org.apache.solr.common.util; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.util.RecordingJSONParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.StringReader; import java.lang.invoke.MethodHandles; @@ -31,6 +26,12 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import org.apache.commons.lang.StringUtils; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.util.RecordingJSONParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class TestJsonRecordReader extends SolrTestCaseJ4 { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -129,18 +130,32 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 { " \"nested_inside\" : \"check check check 1\"\n" + " }\n" + "}"; + String json2 = " {\n" + " \"id\" : \"345\",\n" + + " \"payload\": \""+ StringUtils.repeat("0123456789", 819) + + "\",\n" + " \"description\": \"Testing /json/docs srcField 2\",\n" + "\n" + " \"nested_data\" : {\n" + " \"nested_inside\" : \"check check check 2\"\n" + " }\n" + "}"; - JsonRecordReader streamer = JsonRecordReader.getInst("/", Arrays.asList("id:/id")); - RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json + json2)); + String json3 = + " {\n" + + " \"id\" : \"678\",\n" + + " \"description\": \"Testing /json/docs srcField 3\",\n" + + "\n" + + " \"nested_data\" : {\n" + + " \"nested_inside\" : \"check check check 3\"\n" + + " }\n" + + "}"; + + + JsonRecordReader streamer = JsonRecordReader.getInst("/", Arrays.asList("id:/id")); + RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json + json2 + json3)); streamer.streamRecords(parser, new JsonRecordReader.Handler() { int count = 0; @@ -162,6 +177,12 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 { assertEquals(m.get("description"), "Testing /json/docs srcField 2"); assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 2"); } + if (count++ == 3) { + assertEquals(m.get("id"), "678"); + assertEquals(m.get("description"), "Testing /json/docs srcField 3"); + assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 3"); + } + } });