Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-9768 RecordingJsonParser produces incomplete json (Wojciech Stryszyk via ab)

This commit is contained in:
Andrzej Bialecki 2016-11-29 21:11:40 +01:00
parent 02c687758e
commit 590d31f311
3 changed files with 45 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -216,6 +216,8 @@ Bug Fixes
* SOLR-5260: Facet search on a docvalue field in a multi shard collection (Trym Møller, Erick Erickson)
* SOLR-9768: RecordingJsonParser produces incomplete json (Wojciech Stryszyk via ab)
Other Changes
----------------------

17
solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java
View File

@ -29,7 +29,9 @@ public class RecordingJSONParser extends JSONParser {
private StringBuilder sb = new StringBuilder();
private boolean objectStarted = false;
public long lastMarkedPosition = 0;
private long lastMarkedPosition = 0;
private long lastGlobalPosition = 0;
private static final int BUFFER_SIZE = 8192;
public RecordingJSONParser(Reader in) {
@ -39,7 +41,7 @@ public class RecordingJSONParser extends JSONParser {
}
static char[] getChars() {
buf.set(new char[8192]);
buf.set(new char[BUFFER_SIZE]);
return buf.get();
}
@ -68,11 +70,22 @@ public class RecordingJSONParser extends JSONParser {
if(currPosition < 0){
System.out.println("ERROR");
}
if (currPosition > lastMarkedPosition) {
for (long i = lastMarkedPosition; i < currPosition; i++) {
recordChar(bufCopy[(int) i]);
}
} else if (currPosition < lastMarkedPosition) {
for (long i = 0; i < currPosition; i++) {
recordChar(bufCopy[(int) i]);
}
} else if (currPosition == BUFFER_SIZE && lastGlobalPosition != globalPosition) {
for (long i = 0; i < currPosition; i++) {
recordChar(bufCopy[(int) i]);
}
}
lastGlobalPosition = globalPosition;
lastMarkedPosition = currPosition;
}

35
solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
View File

@ -16,11 +16,6 @@
*/
package org.apache.solr.common.util;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.util.RecordingJSONParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.StringReader;
import java.lang.invoke.MethodHandles;
@ -31,6 +26,12 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.util.RecordingJSONParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestJsonRecordReader extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -129,18 +130,32 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
" \"nested_inside\" : \"check check check 1\"\n" +
" }\n" +
"}";
String json2 =
" {\n" +
" \"id\" : \"345\",\n" +
" \"payload\": \""+ StringUtils.repeat("0123456789", 819) +
"\",\n" +
" \"description\": \"Testing /json/docs srcField 2\",\n" +
"\n" +
" \"nested_data\" : {\n" +
" \"nested_inside\" : \"check check check 2\"\n" +
" }\n" +
"}";
JsonRecordReader streamer = JsonRecordReader.getInst("/", Arrays.asList("id:/id"));
RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json + json2));
String json3 =
" {\n" +
" \"id\" : \"678\",\n" +
" \"description\": \"Testing /json/docs srcField 3\",\n" +
"\n" +
" \"nested_data\" : {\n" +
" \"nested_inside\" : \"check check check 3\"\n" +
" }\n" +
"}";
JsonRecordReader streamer = JsonRecordReader.getInst("/", Arrays.asList("id:/id"));
RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json + json2 + json3));
streamer.streamRecords(parser, new JsonRecordReader.Handler() {
int count = 0;
@ -162,6 +177,12 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
assertEquals(m.get("description"), "Testing /json/docs srcField 2");
assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 2");
}
if (count++ == 3) {
assertEquals(m.get("id"), "678");
assertEquals(m.get("description"), "Testing /json/docs srcField 3");
assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 3");
}
}
});