Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-8582 : memory leak in JsonRecordReader affecting /update/json/docs. Large payloads 

cause OOM

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1726261 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Noble Paul 2016-01-22 17:28:27 +00:00
parent 860a5847b7
commit 9298da5690
3 changed files with 35 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -420,6 +420,9 @@ Bug Fixes
* SOLR-8371: Try and prevent too many recovery requests from stacking up and clean up some faulty
cancel recovery logic. (Mark Miller)
* SOLR-8582 : memory leak in JsonRecordReader affecting /update/json/docs. Large payloads
cause OOM (noble, shalin)
Optimizations
----------------------

18
solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java
View File

@ -136,8 +136,7 @@ public class JsonRecordReader {
public void streamRecords(JSONParser parser, Handler handler) throws IOException {
rootNode.parse(parser, handler,
new LinkedHashMap<String, Object>(),
new Stack<Set<String>>(), false);
new LinkedHashMap<String, Object>());
}
@ -277,22 +276,21 @@ public class JsonRecordReader {
private void parse(JSONParser parser,
Handler handler,
Map<String, Object> values,
Stack<Set<String>> stack, // lists of values to purge
boolean recordStarted) throws IOException {
Map<String, Object> values) throws IOException {
int event = -1;
boolean recordStarted = false;
for (; ; ) {
event = parser.nextEvent();
if (event == EOF) break;
if (event == OBJECT_START) {
handleObjectStart(parser, new HashSet<Node>(), handler, values, stack, recordStarted, null);
handleObjectStart(parser, handler, values, new Stack<Set<String>>(), recordStarted, null);
} else if (event == ARRAY_START) {
for (; ; ) {
event = parser.nextEvent();
if (event == ARRAY_END) break;
if (event == OBJECT_START) {
handleObjectStart(parser, new HashSet<Node>(), handler, values, stack, recordStarted, null);
handleObjectStart(parser, handler, values, new Stack<Set<String>>(), recordStarted, null);
}
}
}
@ -312,7 +310,7 @@ public class JsonRecordReader {
* any inner chil tags are compared against the cache and jumped to if
* matched.
*/
private void handleObjectStart(final JSONParser parser, final Set<Node> childrenFound,
private void handleObjectStart(final JSONParser parser,
final Handler handler, final Map<String, Object> values,
final Stack<Set<String>> stack, boolean recordStarted,
MethodFrameWrapper frameWrapper)
@ -342,13 +340,13 @@ public class JsonRecordReader {
@Override
public void walk(int event) throws IOException {
if (event == OBJECT_START) {
node.handleObjectStart(parser, childrenFound, handler, values, stack, isRecordStarted, this);
node.handleObjectStart(parser, handler, values, stack, isRecordStarted, this);
} else if (event == ARRAY_START) {
for (; ; ) {
event = parser.nextEvent();
if (event == ARRAY_END) break;
if (event == OBJECT_START) {
node.handleObjectStart(parser, childrenFound, handler, values, stack, isRecordStarted, this);
node.handleObjectStart(parser, handler, values, stack, isRecordStarted, this);
}
}
}

24
solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
View File

@ -22,10 +22,12 @@ import org.apache.solr.util.RecordingJSONParser;
import java.io.IOException;
import java.io.StringReader;
import java.lang.ref.WeakReference;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
public class TestJsonRecordReader extends SolrTestCaseJ4 {
@ -256,6 +258,28 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
}
}
public void testArrayOfRootObjects() throws Exception {
String json = "[{'fieldA':'A1'}, {'fieldB':'B2'}]";
JsonRecordReader streamer;
List<Map<String, Object>> records;
final AtomicReference<WeakReference<String>> ref = new AtomicReference<>();
streamer = JsonRecordReader.getInst("/", Collections.singletonList("$FQN:/**"));
streamer.streamRecords(new StringReader(json), new JsonRecordReader.Handler() {
@Override
public void handle(Map<String, Object> record, String path) {
System.gc();
if (ref.get() != null) {
assertNull("This reference is still intact :" +ref.get().get() ,ref.get().get());
}
String fName = record.keySet().iterator().next();
ref.set(new WeakReference<String>(fName));
}
});
}
public void testAIOOBE() throws IOException {
String json = "[ {\n" +
" \"taxon_group\" : {\n" +