From 21fe4164de7b065208bef86026280083bf78b87a Mon Sep 17 00:00:00 2001 From: user Date: Thu, 14 Jun 2018 12:14:49 -0400 Subject: [PATCH] SOLR-12362: Uploading docs in JSON now supports child documents as field values --- solr/CHANGES.txt | 4 + .../solr/handler/loader/JsonLoader.java | 167 +++++---- .../apache/solr/handler/JsonLoaderTest.java | 318 +++++++++++++----- .../apache/solr/common/SolrInputField.java | 4 +- .../solr/common/params/CommonParams.java | 6 + .../solr/common/util/JsonRecordReader.java | 24 +- 6 files changed, 354 insertions(+), 169 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 3832d7ec099..170aae3e4ef 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -74,6 +74,10 @@ New Features * SOLR-12474: Add an UpdateRequest Object that implements RequestWriter.ContentWriter (noble) +* SOLR-12362: Uploading docs in JSON now supports child documents as field values, thus providing a label to the + relationship instead of the current "anonymous" relationship. Use of this experimental feature requires + anonChildDocs=false parameter. (Moshe Bla, David Smiley) + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java index b93d5ef18bb..67628e540f0 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java @@ -93,11 +93,16 @@ public class JsonLoader extends ContentStreamLoader { protected JSONParser parser; protected final int commitWithin; protected final boolean overwrite; + protected final boolean anonChildDoc; + /** + * {@link CommonParams#ANONYMOUS_CHILD_DOCS} Defaults to true. + */ public SingleThreadedJsonLoader(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor processor) { this.processor = processor; this.req = req; this.rsp = rsp; + this.anonChildDoc = req.getParams().getBool(CommonParams.ANONYMOUS_CHILD_DOCS, true); commitWithin = req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1); overwrite = req.getParams().getBool(UpdateParams.OVERWRITE, true); @@ -248,14 +253,28 @@ public class JsonLoader extends ContentStreamLoader { private SolrInputDocument buildDoc(Map m) { SolrInputDocument result = new SolrInputDocument(); for (Map.Entry e : m.entrySet()) { - if (e.getKey() == null) {// special case. JsonRecordReader emits child docs with null key + if (mapEntryIsChildDoc(e.getValue())) { // parse child documents if (e.getValue() instanceof List) { List value = (List) e.getValue(); for (Object o : value) { - if (o instanceof Map) result.addChildDocument(buildDoc((Map) o)); + if (o instanceof Map) { + if (anonChildDoc) { + result.addChildDocument(buildDoc((Map) o)); + } else { + // retain the value as a list, even if the list contains a single value. + if(!result.containsKey(e.getKey())) { + result.setField(e.getKey(), new ArrayList<>(1)); + } + result.addField(e.getKey(), buildDoc((Map) o)); + } + } } } else if (e.getValue() instanceof Map) { - result.addChildDocument(buildDoc((Map) e.getValue())); + if (anonChildDoc) { + result.addChildDocument(buildDoc((Map) e.getValue())); + } else { + result.addField(e.getKey(), buildDoc((Map) e.getValue())); + } } } else { result.setField(e.getKey(), e.getValue()); @@ -530,7 +549,7 @@ public class JsonLoader extends ContentStreamLoader { } String fieldName = parser.getString(); - if (fieldName.equals(JsonLoader.CHILD_DOC_KEY)) { + if (anonChildDoc && fieldName.equals(JsonLoader.CHILD_DOC_KEY)) { ev = parser.nextEvent(); assertEvent(ev, JSONParser.ARRAY_START); while ((ev = parser.nextEvent()) != JSONParser.ARRAY_END) { @@ -554,84 +573,79 @@ public class JsonLoader extends ContentStreamLoader { private void parseFieldValue(SolrInputField sif) throws IOException { int ev = parser.nextEvent(); if (ev == JSONParser.OBJECT_START) { - parseExtendedFieldValue(sif, ev); + parseExtendedFieldValue(ev, sif); } else { - Object val = parseNormalFieldValue(ev, sif.getName()); + Object val = parseNormalFieldValue(ev, sif); sif.setValue(val); } } - private void parseExtendedFieldValue(SolrInputField sif, int ev) throws IOException { + /** + * A method to either extract an index time boost (deprecated), a map for atomic update, or a child document. + * firstly, a solr document SolrInputDocument constructed. It is then determined whether the document is indeed a childDocument(if it has a unique field). + * If so, it is added. + * Otherwise the document is looped over as a map, and is then parsed as an Atomic Update if that is the case. + * @param ev json parser event + * @param sif input field to add value to. + * @throws IOException in case of parsing exception. + */ + private void parseExtendedFieldValue(int ev, SolrInputField sif) throws IOException { assert ev == JSONParser.OBJECT_START; + SolrInputDocument extendedSolrDocument = parseDoc(ev); + + if (isChildDoc(extendedSolrDocument)) { + sif.addValue(extendedSolrDocument); + return; + } + Object normalFieldValue = null; Map extendedInfo = null; - for (; ; ) { - ev = parser.nextEvent(); - switch (ev) { - case JSONParser.STRING: - String label = parser.getString(); - if ("boost".equals(label)) { - ev = parser.nextEvent(); - if (ev != JSONParser.NUMBER && - ev != JSONParser.LONG && - ev != JSONParser.BIGNUMBER) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Boost should have number. " - + "Unexpected " + JSONParser.getEventString(ev) + " at [" + parser.getPosition() + "], field=" + sif.getName()); - } + for (SolrInputField entry: extendedSolrDocument) { + Object val = entry.getValue(); + String label = entry.getName(); + if ("boost".equals(label)) { + Object boostVal = val; + if (!(boostVal instanceof Double)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Boost should have number. " + + "Unexpected value: " + boostVal.toString() + "field=" + label); + } - String message = "Ignoring field boost: " + parser.getDouble() + " as index-time boosts are not supported anymore"; - if (WARNED_ABOUT_INDEX_TIME_BOOSTS.compareAndSet(false, true)) { - log.warn(message); - } else { - log.debug(message); - } - } else if ("value".equals(label)) { - normalFieldValue = parseNormalFieldValue(parser.nextEvent(), sif.getName()); - } else { - // If we encounter other unknown map keys, then use a map - if (extendedInfo == null) { - extendedInfo = new HashMap<>(2); - } - // for now, the only extended info will be field values - // we could either store this as an Object or a SolrInputField - Object val = parseNormalFieldValue(parser.nextEvent(), sif.getName()); - extendedInfo.put(label, val); - } - break; - - case JSONParser.OBJECT_END: - if (extendedInfo != null) { - if (normalFieldValue != null) { - extendedInfo.put("value", normalFieldValue); - } - sif.setValue(extendedInfo); - } else { - sif.setValue(normalFieldValue); - } - return; - - default: - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing JSON extended field value. " - + "Unexpected " + JSONParser.getEventString(ev) + " at [" + parser.getPosition() + "], field=" + sif.getName()); + String message = "Ignoring field boost: " + boostVal.toString() + " as index-time boosts are not supported anymore"; + if (WARNED_ABOUT_INDEX_TIME_BOOSTS.compareAndSet(false, true)) { + log.warn(message); + } else { + log.debug(message); + } + } else if ("value".equals(label)) { + normalFieldValue = val; + } else { + // If we encounter other unknown map keys, then use a map + if (extendedInfo == null) { + extendedInfo = new HashMap<>(2); + } + // for now, the only extended info will be field values + // we could either store this as an Object or a SolrInputField + extendedInfo.put(label, val); + } + if (extendedInfo != null) { + if (normalFieldValue != null) { + extendedInfo.put("value", normalFieldValue); + } + sif.setValue(extendedInfo); + } else { + sif.setValue(normalFieldValue); } } } - private Object parseNormalFieldValue(int ev, String fieldName) throws IOException { - if (ev == JSONParser.ARRAY_START) { - List val = parseArrayFieldValue(ev, fieldName); - return val; - } else { - Object val = parseSingleFieldValue(ev, fieldName); - return val; - } + private Object parseNormalFieldValue(int ev, SolrInputField sif) throws IOException { + return ev == JSONParser.ARRAY_START ? parseArrayFieldValue(ev, sif): parseSingleFieldValue(ev, sif); } - - private Object parseSingleFieldValue(int ev, String fieldName) throws IOException { + private Object parseSingleFieldValue(int ev, SolrInputField sif) throws IOException { switch (ev) { case JSONParser.STRING: return parser.getString(); @@ -647,15 +661,18 @@ public class JsonLoader extends ContentStreamLoader { parser.getNull(); return null; case JSONParser.ARRAY_START: - return parseArrayFieldValue(ev, fieldName); + return parseArrayFieldValue(ev, sif); + case JSONParser.OBJECT_START: + parseExtendedFieldValue(ev, sif); + return sif.getValue(); default: throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing JSON field value. " - + "Unexpected " + JSONParser.getEventString(ev) + " at [" + parser.getPosition() + "], field=" + fieldName); + + "Unexpected " + JSONParser.getEventString(ev) + " at [" + parser.getPosition() + "], field=" + sif.getName()); } } - private List parseArrayFieldValue(int ev, String fieldName) throws IOException { + private List parseArrayFieldValue(int ev, SolrInputField sif) throws IOException { assert ev == JSONParser.ARRAY_START; ArrayList lst = new ArrayList(2); @@ -664,10 +681,24 @@ public class JsonLoader extends ContentStreamLoader { if (ev == JSONParser.ARRAY_END) { return lst; } - Object val = parseSingleFieldValue(ev, fieldName); + Object val = parseSingleFieldValue(ev, sif); lst.add(val); + sif.setValue(null); } } + + private boolean isChildDoc(SolrInputDocument extendedFieldValue) { + return extendedFieldValue.containsKey(req.getSchema().getUniqueKeyField().getName()); + } + + private boolean mapEntryIsChildDoc(Object val) { + if(val instanceof List) { + List listVal = (List) val; + if (listVal.size() == 0) return false; + return listVal.get(0) instanceof Map; + } + return val instanceof Map; + } } private static Object changeChildDoc(Object o) { diff --git a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java index 9ffa71fb9e1..fdfeae44336 100644 --- a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java +++ b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java @@ -16,6 +16,11 @@ */ package org.apache.solr.handler; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.UnaryOperator; + import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; @@ -32,10 +37,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.noggit.ObjectBuilder; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.UnaryOperator; +import static org.apache.solr.common.params.CommonParams.ANONYMOUS_CHILD_DOCS; public class JsonLoaderTest extends SolrTestCaseJ4 { @@ -411,6 +413,7 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { " \"id\": \"1.2\",\n" + " \"name\": \"i am the 2nd child\",\n" + " \"cat\": \"child\",\n" + + " \"test_s\": \"test-new-label\",\n" + " \"grandchildren\": [\n" + " {\n" + " \"id\": \"1.2.1\",\n" + @@ -432,63 +435,87 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { "f", "name:/children/grandchildren/name", "f", "cat:/children/grandchildren/cat"}; + @Test + public void testFewParentsAnonymousJsonDoc() throws Exception { + String json = PARENT_TWO_CHILDREN_JSON; + assertFewParents(json, true); + } + + @Test public void testFewParentsJsonDoc() throws Exception { String json = PARENT_TWO_CHILDREN_JSON; + assertFewParents(json, false); + } + + private void assertFewParents(String json, boolean anonChildDocsFlag) throws Exception { SolrQueryRequest req; SolrQueryResponse rsp; BufferingRequestProcessor p; - JsonLoader loader; - { //multichild test case - final boolean array = random().nextBoolean(); - StringBuilder b = new StringBuilder(); + JsonLoader loader;//multichild test case + final boolean array = random().nextBoolean(); + StringBuilder b = new StringBuilder(); + if (array) { + b.append("["); + } + final int passes = atLeast(2); + for (int i=1;i<=passes;i++){ + b.append(json.replace("1",""+i)); if (array) { - b.append("["); + b.append(i s = (v)-> v.replace("1",""+ii); + final SolrInputDocument parent = p.addCommands.get(i-1).solrDoc; + assertOnlyValue(s.apply("1"), parent,"id"); + assertOnlyValue("i am the parent", parent, "name"); + assertOnlyValue("parent", parent, "cat"); + + List childDocs1; + if(anonChildDocsFlag) { + childDocs1 = parent.getChildDocuments(); + } else { + childDocs1 = (List) ((parent.getField("children")).getValue()); } - - req = req(PARENT_TWO_CHILDREN_PARAMS); - req.getContext().put("path", "/update/json/docs"); - rsp = new SolrQueryResponse(); - p = new BufferingRequestProcessor(null); - loader = new JsonLoader(); - loader.load(req, rsp, new ContentStreamBase.StringStream(b.toString()), p); - for (int i=1; i<=passes; i++){ - final int ii = i; - UnaryOperator s = (v)-> v.replace("1",""+ii); - final SolrInputDocument parent = p.addCommands.get(i-1).solrDoc; - assertOnlyValue(s.apply("1"), parent,"id"); - assertOnlyValue("i am the parent", parent, "name"); - assertOnlyValue("parent", parent, "cat"); - - assertEquals(2, parent.getChildDocuments().size()); - { - final SolrInputDocument child1 = parent.getChildDocuments().get(0); - assertOnlyValue(s.apply("1.1"), child1, "id"); - assertOnlyValue(s.apply("i am the 1st child"), child1, "name"); - assertOnlyValue("child", child1,"cat"); - } - { - final SolrInputDocument child2 = parent.getChildDocuments().get(1); - assertOnlyValue(s.apply("1.2"), child2, "id"); - assertOnlyValue("i am the 2nd child", child2, "name"); - assertOnlyValue("child", child2, "cat"); - - assertEquals(1, child2.getChildDocuments().size()); - final SolrInputDocument grandChild = child2.getChildDocuments().get(0); - assertOnlyValue(s.apply("1.2.1"), grandChild,"id"); - assertOnlyValue("i am the grandchild", grandChild, "name"); - assertOnlyValue("grandchild", grandChild, "cat"); + + assertEquals(2, childDocs1.size()); + { + final SolrInputDocument child1 = childDocs1.get(0); + assertOnlyValue(s.apply("1.1"), child1, "id"); + assertOnlyValue(s.apply("i am the 1st child"), child1, "name"); + assertOnlyValue("child", child1,"cat"); + } + { + final SolrInputDocument child2 = childDocs1.get(1); + assertOnlyValue(s.apply("1.2"), child2, "id"); + assertOnlyValue("i am the 2nd child", child2, "name"); + assertOnlyValue("test-new-label", child2, "test_s"); + assertOnlyValue("child", child2, "cat"); + + List childDocs2; + if(anonChildDocsFlag) { + childDocs2 = child2.getChildDocuments(); + } else { + childDocs2 = (List) ((child2.getField("grandchildren")).getValue()); } + + assertEquals(1, childDocs2.size()); + final SolrInputDocument grandChild = childDocs2.get(0); + assertOnlyValue(s.apply("1.2.1"), grandChild,"id"); + assertOnlyValue("i am the grandchild", grandChild, "name"); + assertOnlyValue("grandchild", grandChild, "cat"); } } } - + private static void assertOnlyValue(String expected, SolrInputDocument doc, String field) { assertEquals(Collections.singletonList(expected), doc.getFieldValues(field)); } @@ -790,53 +817,69 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { req.close(); } + private static final String SIMPLE_JSON_CHILD_DOCS = "{\n" + + " \"add\": {\n" + + " \"doc\": {\n" + + " \"id\": \"1\",\n" + + " \"_childDocuments_\": [\n" + + " {\n" + + " \"id\": \"2\"\n" + + " },\n" + + " {\n" + + " \"id\": \"3\",\n" + + " \"foo_i\": [666,777]\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; + + @Test + public void testSimpleAnonymousChildDocs() throws Exception { + checkTwoAnonymousChildDocs(SIMPLE_JSON_CHILD_DOCS); + } + @Test public void testSimpleChildDocs() throws Exception { - String str = "{\n" + - " \"add\": {\n" + - " \"doc\": {\n" + - " \"id\": \"1\",\n" + - " \"_childDocuments_\": [\n" + - " {\n" + - " \"id\": \"2\"\n" + - " },\n" + - " {\n" + - " \"id\": \"3\",\n" + - " \"foo_i\": [666,777]\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - "}"; - checkTwoChildDocs(str); + checkTwoAnonymousChildDocs(SIMPLE_JSON_CHILD_DOCS, false); + } + + private static final String DUP_KEYS_JSON_CHILD_DOCS = "{\n" + + " \"add\": {\n" + + " \"doc\": {\n" + + " \"_childDocuments_\": [\n" + + " {\n" + + " \"id\": \"2\"\n" + + " }\n" + + " ],\n" + + " \"id\": \"1\",\n" + + " \"_childDocuments_\": [\n" + + " {\n" + + " \"id\": \"3\",\n" + + " \"foo_i\": 666,\n" + + " \"foo_i\": 777\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; + + @Test + public void testDupKeysAnonymousChildDocs() throws Exception { + checkTwoAnonymousChildDocs(DUP_KEYS_JSON_CHILD_DOCS); } @Test public void testDupKeysChildDocs() throws Exception { - String str = "{\n" + - " \"add\": {\n" + - " \"doc\": {\n" + - " \"_childDocuments_\": [\n" + - " {\n" + - " \"id\": \"2\"\n" + - " }\n" + - " ],\n" + - " \"id\": \"1\",\n" + - " \"_childDocuments_\": [\n" + - " {\n" + - " \"id\": \"3\",\n" + - " \"foo_i\": 666,\n" + - " \"foo_i\": 777\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - "}"; - checkTwoChildDocs(str); + checkTwoAnonymousChildDocs(DUP_KEYS_JSON_CHILD_DOCS, false); } - private void checkTwoChildDocs(String rawJsonStr) throws Exception { - SolrQueryRequest req = req("commit","true"); + private void checkTwoAnonymousChildDocs(String rawJsonStr) throws Exception { + checkTwoAnonymousChildDocs(rawJsonStr, true); + } + + private void checkTwoAnonymousChildDocs(String rawJsonStr, boolean anonChildDocs) throws Exception { + SolrQueryRequest req = req("commit","true", ANONYMOUS_CHILD_DOCS, Boolean.toString(anonChildDocs)); SolrQueryResponse rsp = new SolrQueryResponse(); BufferingRequestProcessor p = new BufferingRequestProcessor(null); JsonLoader loader = new JsonLoader(); @@ -849,11 +892,20 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { SolrInputField f = d.getField( "id" ); assertEquals("1", f.getValue()); - SolrInputDocument cd = d.getChildDocuments().get(0); + SolrInputDocument cd; + if (anonChildDocs) { + cd = d.getChildDocuments().get(0); + } else { + cd = (SolrInputDocument) (d.getField("_childDocuments_")).getFirstValue(); + } SolrInputField cf = cd.getField( "id" ); assertEquals("2", cf.getValue()); - cd = d.getChildDocuments().get(1); + if (anonChildDocs) { + cd = d.getChildDocuments().get(1); + } else { + cd = (SolrInputDocument)((List)(d.getField("_childDocuments_")).getValue()).get(1); + } cf = cd.getField( "id" ); assertEquals("3", cf.getValue()); cf = cd.getField( "foo_i" ); @@ -865,7 +917,7 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { } @Test - public void testEmptyChildDocs() throws Exception { + public void testEmptyAnonymousChildDocs() throws Exception { String str = "{\n" + " \"add\": {\n" + " \"doc\": {\n" + @@ -893,7 +945,7 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { } @Test - public void testGrandChildDocs() throws Exception { + public void testAnonymousGrandChildDocs() throws Exception { String str = "{\n" + " \"add\": {\n" + " \"doc\": {\n" + @@ -946,5 +998,87 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { } + @Test + public void testChildDocs() throws Exception { + String str = "{\n" + + " \"add\": {\n" + + " \"doc\": {\n" + + " \"id\": \"1\",\n" + + " \"children\": [\n" + + " {\n" + + " \"id\": \"2\",\n" + + " \"foo_s\": \"Yaz\"\n" + + " },\n" + + " {\n" + + " \"id\": \"3\",\n" + + " \"foo_s\": \"Bar\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; + + SolrQueryRequest req = req("commit","true"); + SolrQueryResponse rsp = new SolrQueryResponse(); + BufferingRequestProcessor p = new BufferingRequestProcessor(null); + JsonLoader loader = new JsonLoader(); + loader.load(req, rsp, new ContentStreamBase.StringStream(str), p); + + assertEquals( 1, p.addCommands.size() ); + + AddUpdateCommand add = p.addCommands.get(0); + SolrInputDocument one = add.solrDoc; + assertEquals("1", one.getFieldValue("id")); + + List children = (List) one.getFieldValues("children"); + SolrInputDocument two = children.get(0); + assertEquals("2", two.getFieldValue("id")); + assertEquals("Yaz", two.getFieldValue("foo_s")); + + SolrInputDocument three = children.get(1); + assertEquals("3", three.getFieldValue("id")); + assertEquals("Bar", three.getFieldValue("foo_s")); + + req.close(); + + } + + @Test + public void testSingleRelationalChildDoc() throws Exception { + String str = "{\n" + + " \"add\": {\n" + + " \"doc\": {\n" + + " \"id\": \"1\",\n" + + " \"child1\": \n" + + " {\n" + + " \"id\": \"2\",\n" + + " \"foo_s\": \"Yaz\"\n" + + " },\n" + + " }\n" + + " }\n" + + "}"; + + SolrQueryRequest req = req("commit","true"); + SolrQueryResponse rsp = new SolrQueryResponse(); + BufferingRequestProcessor p = new BufferingRequestProcessor(null); + JsonLoader loader = new JsonLoader(); + loader.load(req, rsp, new ContentStreamBase.StringStream(str), p); + + assertEquals( 1, p.addCommands.size() ); + + AddUpdateCommand add = p.addCommands.get(0); + SolrInputDocument one = add.solrDoc; + assertEquals("1", one.getFieldValue("id")); + + assertTrue(one.keySet().contains("child1")); + + SolrInputDocument two = (SolrInputDocument) one.getField("child1").getValue(); + assertEquals("2", two.getFieldValue("id")); + assertEquals("Yaz", two.getFieldValue("foo_s")); + + req.close(); + + } + } diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java b/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java index 94e98de7b77..5d99d92d92c 100644 --- a/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java +++ b/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java @@ -87,8 +87,8 @@ public class SolrInputField implements Iterable, Serializable value = vals; } - // Add the new values to a collection - if( v instanceof Iterable ) { + // Add the new values to a collection, if childDoc add as is without iteration + if( v instanceof Iterable && !(v instanceof SolrDocumentBase)) { for( Object o : (Iterable)v ) { vals.add( o ); } diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java index 21879ba174c..054e9eee804 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java @@ -290,5 +290,11 @@ public interface CommonParams { String JSON_MIME = "application/json"; String JAVABIN_MIME = "application/javabin"; + + /** + * If set to true, child documents will be added as anonymous children into the _childDocuments list, + * else, child documents will be added to SolrInputDocument as field values according to their key name. + */ + String ANONYMOUS_CHILD_DOCS = "anonChildDocs"; } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java b/solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java index 24adb9078db..7f93bc4691a 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java @@ -288,13 +288,13 @@ public class JsonRecordReader { event = parser.nextEvent(); if (event == EOF) break; if (event == OBJECT_START) { - handleObjectStart(parser, handler, values, new Stack<>(), recordStarted, null); + handleObjectStart(parser, handler, new LinkedHashMap<>(), new Stack<>(), recordStarted, null); } else if (event == ARRAY_START) { for (; ; ) { event = parser.nextEvent(); if (event == ARRAY_END) break; if (event == OBJECT_START) { - handleObjectStart(parser, handler, values, new Stack<>(), recordStarted, null); + handleObjectStart(parser, handler, new LinkedHashMap<>(), new Stack<>(), recordStarted, null); } } } @@ -350,6 +350,10 @@ public class JsonRecordReader { event = parser.nextEvent(); if (event == ARRAY_END) break; if (event == OBJECT_START) { + // if single item in array will still be added as array + if(!values.containsKey(name)) { + values.put(name, new ArrayList<>()); + } walkObject(); } } @@ -359,7 +363,7 @@ public class JsonRecordReader { void walkObject() throws IOException { if (node.isChildRecord) { node.handleObjectStart(parser, - (record, path) -> addChildDoc2ParentDoc(record, values), + (record, path) -> addChildDoc2ParentDoc(record, values, getPathSuffix(path)), new LinkedHashMap<>(), new Stack<>(), true, @@ -438,18 +442,18 @@ public class JsonRecordReader { } } - private void addChildDoc2ParentDoc(Map record, Map values) { + private void addChildDoc2ParentDoc(Map record, Map values, String key) { record = Utils.getDeepCopy(record, 2); - Object oldVal = values.get(null); + Object oldVal = values.get(key); if (oldVal == null) { - values.put(null, record); + values.put(key, record); } else if (oldVal instanceof List) { ((List) oldVal).add(record); } else { ArrayList l = new ArrayList(); l.add(oldVal); l.add(record); - values.put(null, l); + values.put(key, l); } } @@ -486,6 +490,12 @@ public class JsonRecordReader { values.put(fieldName, l); } + // returns the last key of the path + private String getPathSuffix(String path) { + int indexOf = path.lastIndexOf("/"); + if (indexOf == -1) return path; + return path.substring(indexOf + 1); + } @Override public String toString() {