From 73d5f1c52cd7a9531f07aea6c9f88d1ff253ac64 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Tue, 21 Jun 2016 18:50:49 +0530 Subject: [PATCH] SOLR-9234: srcField parameter works only when all fields are captured in the /update/json/docs endpoint --- solr/CHANGES.txt | 3 + .../apache/solr/util/RecordingJSONParser.java | 79 +++++++++++-------- .../common/util/TestJsonRecordReader.java | 52 ++++++++++++ 3 files changed, 103 insertions(+), 31 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a2c46d226ca..e950f1014a9 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -47,6 +47,9 @@ Bug Fixes * SOLR-9161: Change SolrPluginUtils.invokeSetters implementation to accommodate setter variants. (Christine Poerschke, Steve Rowe, Uwe Schindler) +* SOLR-9234: srcField parameter works only when all fields are captured in the /update/json/docs + endpoint (noble) + Other Changes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java b/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java index 02ff03c11e1..c2ec2ba20ff 100644 --- a/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java +++ b/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java @@ -22,56 +22,73 @@ import java.io.Reader; import org.noggit.CharArr; import org.noggit.JSONParser; -public class RecordingJSONParser extends JSONParser{ - public RecordingJSONParser(Reader in) { super(in); } +public class RecordingJSONParser extends JSONParser { + static ThreadLocal buf = new ThreadLocal<>(); + private final char[] bufCopy; + //global position is the global position at the beginning of my buffer + private long globalPosition = 0; - private StringBuilder sb = new StringBuilder() ; - private long position; - private boolean objectStarted =false; + private StringBuilder sb = new StringBuilder(); + private boolean objectStarted = false; + public long lastMarkedPosition = 0; + public RecordingJSONParser(Reader in) { + super(in, getChars()); + bufCopy = buf.get(); + buf.remove(); + } - @Override - protected int getChar() throws IOException { - int aChar = super.getChar(); - if(aChar == '{') objectStarted =true; - if(getPosition() >position) recordChar((char) aChar); // check before adding if a pushback happened ignore - position= getPosition(); - return aChar; + static char[] getChars() { + buf.set(new char[8192]); + return buf.get(); } private void recordChar(int aChar) { - if(objectStarted) + if (objectStarted) { sb.append((char) aChar); + } else if (aChar == '{') { + sb.append((char) aChar); + objectStarted = true; + } } - private void recordStr(String s) { - if(objectStarted) sb.append(s); + + public void resetBuf() { + sb = new StringBuilder(); + objectStarted = false; } @Override - public CharArr getStringChars() throws IOException { - CharArr chars = super.getStringChars(); - recordStr(chars.toString()); - position = getPosition(); - // if reading a String , the getStringChars do not return the closing single quote or double quote - //so, try to capture that - if(chars.getArray().length >chars.getStart()+chars.size()) { - char next = chars.getArray()[chars.getStart() + chars.size()]; - if(next =='"' || next == '\'') { - recordChar(next); + public int nextEvent() throws IOException { + captureMissing(); + return super.nextEvent(); + } + + private void captureMissing() { + long currPosition = getPosition() - globalPosition; + if(currPosition < 0){ + System.out.println("ERROR"); + } + if (currPosition > lastMarkedPosition) { + for (long i = lastMarkedPosition; i < currPosition; i++) { + recordChar(bufCopy[(int) i]); } } - return chars; + lastMarkedPosition = currPosition; } - public void resetBuf(){ - sb = new StringBuilder(); - objectStarted=false; - } + @Override + protected void fill() throws IOException { + captureMissing(); + super.fill(); + this.globalPosition = getPosition(); + + } public String getBuf() { - if(sb != null) return sb.toString(); + captureMissing(); + if (sb != null) return sb.toString(); return null; } diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java b/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java index b94808c063f..d59dea39146 100644 --- a/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java @@ -18,9 +18,12 @@ package org.apache.solr.common.util; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.util.RecordingJSONParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.StringReader; +import java.lang.invoke.MethodHandles; import java.lang.ref.WeakReference; import java.util.Arrays; import java.util.Collections; @@ -30,6 +33,8 @@ import java.util.concurrent.atomic.AtomicReference; public class TestJsonRecordReader extends SolrTestCaseJ4 { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public void testOneLevelSplit() throws IOException { String json = "{\n" + " \"a\":\"A\" ,\n" + @@ -115,6 +120,53 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 { } + public void testSrcField() throws Exception { + String json = "{\n" + + " \"id\" : \"123\",\n" + + " \"description\": \"Testing /json/docs srcField 1\",\n" + + "\n" + + " \"nested_data\" : {\n" + + " \"nested_inside\" : \"check check check 1\"\n" + + " }\n" + + "}"; + String json2 = + " {\n" + + " \"id\" : \"345\",\n" + + " \"description\": \"Testing /json/docs srcField 2\",\n" + + "\n" + + " \"nested_data\" : {\n" + + " \"nested_inside\" : \"check check check 2\"\n" + + " }\n" + + "}"; + JsonRecordReader streamer = JsonRecordReader.getInst("/", Arrays.asList("id:/id")); + RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json + json2)); + + + streamer.streamRecords(parser, new JsonRecordReader.Handler() { + int count = 0; + + @Override + public void handle(Map record, String path) { + count++; + String buf = parser.getBuf(); + parser.resetBuf(); + + Map m = (Map) Utils.fromJSONString(buf); + if (count == 1) { + assertEquals(m.get("id"), "123"); + assertEquals(m.get("description"), "Testing /json/docs srcField 1"); + assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 1"); + } + if (count++ == 2) { + assertEquals(m.get("id"), "345"); + assertEquals(m.get("description"), "Testing /json/docs srcField 2"); + assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 2"); + } + } + }); + + } + public void testRecursiveWildCard() throws IOException { String json = "{\n" + " \"a\":\"A\" ,\n" +