Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-6633 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1633390 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Noble Paul 2014-10-21 14:53:04 +00:00
parent 2e1c562b4b
commit de911858d9
6 changed files with 298 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
View File

@ -16,21 +16,34 @@ package org.apache.solr.handler.loader;
* limitations under the License.
*/
import java.io.FilterReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.lang.reflect.Field;
import java.nio.CharBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import com.ctc.wstx.stax.FilteredStreamReader;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.TeeInputStream;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.JsonRecordReader;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.xslt.RecordingJSONParser;
import org.noggit.CharArr;
import org.noggit.JSONParser;
import org.noggit.ObjectBuilder;
import org.apache.solr.common.SolrException;
@ -50,12 +63,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @since solr 4.0
*/
public class JsonLoader extends ContentStreamLoader {
final static Logger log = LoggerFactory.getLogger( JsonLoader.class );
final static Logger log = LoggerFactory.getLogger(JsonLoader.class);
private static final String CHILD_DOC_KEY = "_childDocuments_";
@Override
@ -65,8 +77,8 @@ public class JsonLoader extends ContentStreamLoader {
@Override
public void load(SolrQueryRequest req, SolrQueryResponse rsp,
ContentStream stream, UpdateRequestProcessor processor) throws Exception {
new SingleThreadedJsonLoader(req,rsp,processor).load(req, rsp, stream, processor);
ContentStream stream, UpdateRequestProcessor processor) throws Exception {
new SingleThreadedJsonLoader(req, rsp, processor).load(req, rsp, stream, processor);
}
@ -103,24 +115,22 @@ public class JsonLoader extends ContentStreamLoader {
reader = new StringReader(body);
}
parser = new JSONParser(reader);
this.processUpdate();
}
finally {
this.processUpdate(reader);
} finally {
IOUtils.closeQuietly(reader);
}
}
@SuppressWarnings("fallthrough")
void processUpdate() throws IOException
{
void processUpdate(Reader reader) throws IOException {
String path = (String) req.getContext().get("path");
if(UpdateRequestHandler.DOC_PATH.equals(path) || "false".equals( req.getParams().get("json.command"))){
String split = req.getParams().get("split");
String[] f = req.getParams().getParams("f");
handleSplitMode(split,f);
handleSplitMode(split, f, reader);
return;
}
parser = new JSONParser(reader);
int ev = parser.nextEvent();
while( ev != JSONParser.EOF ) {
@ -190,27 +200,41 @@ public class JsonLoader extends ContentStreamLoader {
}
}
private void handleSplitMode(String split, String[] fields) throws IOException {
if(split == null) split = "/";
if(fields == null || fields.length ==0) fields = new String[]{"$FQN:/**"};
final boolean echo = "true".equals( req.getParams().get("echo"));
private void handleSplitMode(String split, String[] fields, final Reader reader) throws IOException {
if (split == null) split = "/";
if (fields == null || fields.length == 0) fields = new String[]{"$FQN:/**"};
final boolean echo = "true".equals(req.getParams().get("echo"));
final String srcField = req.getParams().get("srcField");
final boolean mapUniqueKeyOnly = req.getParams().getBool("mapUniqueKeyOnly",false);
if (srcField != null) {
if (!"/".equals(split))
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Raw data can be stored only if split=/");
parser = new RecordingJSONParser(reader);
} else {
parser = new JSONParser(reader);
}
JsonRecordReader jsonRecordReader = JsonRecordReader.getInst(split, Arrays.asList(fields));
jsonRecordReader.streamRecords(parser,new JsonRecordReader.Handler() {
ArrayList docs =null;
jsonRecordReader.streamRecords(parser, new JsonRecordReader.Handler() {
ArrayList docs = null;
@Override
public void handle(Map<String, Object> record, String path) {
if(echo){
if(docs ==null) {
Map<String, Object> copy = getDocMap(record, parser, srcField, mapUniqueKeyOnly);
if (echo) {
if (docs == null) {
docs = new ArrayList();
rsp.add("docs",docs);
rsp.add("docs", docs);
}
docs.add(record);
docs.add(copy);
} else {
AddUpdateCommand cmd = new AddUpdateCommand(req);
cmd.commitWithin = commitWithin;
cmd.overwrite = overwrite;
cmd.solrDoc = new SolrInputDocument();
for (Map.Entry<String, Object> entry : record.entrySet()) {
cmd.solrDoc = new SolrInputDocument();
for (Map.Entry<String, Object> entry : copy.entrySet()) {
cmd.solrDoc.setField(entry.getKey(),entry.getValue());
}
try {
@ -223,6 +247,37 @@ public class JsonLoader extends ContentStreamLoader {
});
}
private Map<String, Object> getDocMap(Map<String, Object> record, JSONParser parser, String srcField, boolean mapUniqueKeyOnly) {
Map result = record;
if(srcField != null && parser instanceof RecordingJSONParser){
//if srcFIeld specified extract it out first
result = new LinkedHashMap(record);
RecordingJSONParser rjp = (RecordingJSONParser) parser;
result.put(srcField, rjp.getBuf());
rjp.resetBuf();
}
if(mapUniqueKeyOnly){
SchemaField sf = req.getSchema().getUniqueKeyField();
if(sf == null) throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No uniqueKey specified in schema");
String df = req.getParams().get(CommonParams.DF);
if(df == null)throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No 'df' specified in request");
Map copy = new LinkedHashMap();
String uniqueField = (String) record.get(sf.getName());
if(uniqueField == null) uniqueField = UUID.randomUUID().toString().toLowerCase(Locale.ROOT);
copy.put(sf.getName(),uniqueField);
if(srcField != null && result.containsKey(srcField)){
copy.put(srcField, result.remove(srcField));
}
copy.put(df, result.values());
result = copy;
}
return result;
}
/*private void handleStreamingSingleDocs() throws IOException
{
while( true ) {

79
solr/core/src/java/org/apache/solr/util/xslt/RecordingJSONParser.java Normal file
View File

@ -0,0 +1,79 @@
package org.apache.solr.util.xslt;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.Reader;
import org.noggit.CharArr;
import org.noggit.JSONParser;
public class RecordingJSONParser extends JSONParser{
public RecordingJSONParser(Reader in) { super(in); }
private StringBuilder sb = new StringBuilder() ;
private long position;
private boolean objectStarted =false;
@Override
protected int getChar() throws IOException {
int aChar = super.getChar();
if(aChar == '{') objectStarted =true;
if(getPosition() >position) recordChar((char) aChar); // check before adding if a pushback happened ignore
position= getPosition();
return aChar;
}
private void recordChar(int aChar) {
if(objectStarted)
sb.append((char) aChar);
}
private void recordStr(String s) {
if(objectStarted) sb.append(s);
}
@Override
public CharArr getStringChars() throws IOException {
CharArr chars = super.getStringChars();
recordStr(chars.toString());
position = getPosition();
// if reading a String , the getStringChars do not return the closing single quote or double quote
//so, try to capture that
if(chars.getArray().length >=chars.getStart()+chars.size()) {
char next = chars.getArray()[chars.getStart() + chars.size()];
if(next =='"' || next == '\'') {
recordChar(next);
}
}
return chars;
}
public void resetBuf(){
sb = new StringBuilder();
objectStarted=false;
}
public String getBuf() {
if(sb != null) return sb.toString();
return null;
}
}

50
solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
View File

@ -31,6 +31,7 @@ import org.apache.solr.update.DeleteUpdateCommand;
import org.apache.solr.update.processor.BufferingRequestProcessor;
import org.junit.BeforeClass;
import org.junit.Test;
import org.noggit.ObjectBuilder;
import org.xml.sax.SAXException;
import java.math.BigDecimal;
@ -276,7 +277,8 @@ public class JsonLoaderTest extends SolrTestCaseJ4 {
" \"f1\": \"v2\",\n" +
" \"f2\": null\n" +
" }\n";
SolrQueryRequest req = req("json.command","false");
SolrQueryRequest req = req("srcField","_src");
req.getContext().put("path","/update/json/docs");
SolrQueryResponse rsp = new SolrQueryResponse();
BufferingRequestProcessor p = new BufferingRequestProcessor(null);
JsonLoader loader = new JsonLoader();
@ -284,25 +286,26 @@ public class JsonLoaderTest extends SolrTestCaseJ4 {
assertEquals( 2, p.addCommands.size() );
doc = "\n" +
"\n" +
"{bool: true,\n" +
" f0: \"v0\",\n" +
" f2: {\n" +
"{\"bool\": true,\n" +
" \"f0\": \"v0\",\n" +
" \"f2\": {\n" +
" \t \"boost\": 2.3,\n" +
" \t \"value\": \"test\"\n" +
" \t },\n" +
"array: [ \"aaa\", \"bbb\" ],\n" +
"boosted: {\n" +
"\"array\": [ \"aaa\", \"bbb\" ],\n" +
"\"boosted\": {\n" +
" \t \"boost\": 6.7,\n" +
" \t \"value\": [ \"aaa\", \"bbb\" ]\n" +
" \t }\n" +
" }\n" +
"\n" +
"\n" +
" {f1: \"v1\",\n" +
" f1: \"v2\",\n" +
" f2: null\n" +
" {\"f1\": \"v1\",\n" +
" \"f2\": \"v2\",\n" +
" \"f3\": null\n" +
" }\n";
req = req("json.command","false");
req = req("srcField","_src");
req.getContext().put("path","/update/json/docs");
rsp = new SolrQueryResponse();
p = new BufferingRequestProcessor(null);
loader = new JsonLoader();
@ -310,15 +313,40 @@ public class JsonLoaderTest extends SolrTestCaseJ4 {
assertEquals( 2, p.addCommands.size() );
String content = (String) p.addCommands.get(0).solrDoc.getFieldValue("_src");
assertNotNull(content);
Map obj = (Map) ObjectBuilder.fromJSON(content);
assertEquals(Boolean.TRUE, obj.get("bool"));
assertEquals("v0", obj.get("f0"));
assertNotNull(obj.get("f0"));
assertNotNull(obj.get("array"));
assertNotNull(obj.get("boosted"));
content = (String) p.addCommands.get(1).solrDoc.getFieldValue("_src");
assertNotNull(content);
obj = (Map) ObjectBuilder.fromJSON(content);
assertEquals("v1", obj.get("f1"));
assertEquals("v2", obj.get("f2"));
assertTrue(obj.containsKey("f3"));
doc = "[{'id':'1'},{'id':'2'}]".replace('\'', '"');
req = req("json.command","false");
req = req("srcField","_src");
req.getContext().put("path","/update/json/docs");
rsp = new SolrQueryResponse();
p = new BufferingRequestProcessor(null);
loader = new JsonLoader();
loader.load(req, rsp, new ContentStreamBase.StringStream(doc), p);
assertEquals( 2, p.addCommands.size() );
content = (String) p.addCommands.get(0).solrDoc.getFieldValue("_src");
assertNotNull(content);
obj = (Map) ObjectBuilder.fromJSON(content);
assertEquals("1", obj.get("id"));
content = (String) p.addCommands.get(1).solrDoc.getFieldValue("_src");
assertNotNull(content);
obj = (Map) ObjectBuilder.fromJSON(content);
assertEquals("2", obj.get("id"));
}

1
solr/example/solr/collection1/conf/schema.xml
View File

@ -156,6 +156,7 @@
<field name="content_type" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="last_modified" type="date" indexed="true" stored="true"/>
<field name="links" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="_src" type="string" indexed="false" stored="true"/>
<!-- Main body of document extracted by SolrCell.
NOTE: This field is not indexed by default, since it is also copied to "text"

13
solr/example/solr/collection1/conf/solrconfig.xml
View File

@ -1057,6 +1057,19 @@
<str name="df">text</str>
</lst>
</initParams>
<initParams path="/update/json/docs">
<lst name="defaults">
<!--this ensures that the entire json doc will be stored verbatim into one field-->
<str name="srcField">_src</str>
<!--This means a the uniqueKeyField will be extracted from the fields and
all fields go into the 'df' field. In this config df is already configured to be 'text'
-->
<str name="mapUniqueKeyOnly">true</str>
</lst>
</initParams>
<!-- The following are implicitly added
<requestHandler name="/update/json" class="solr.UpdateRequestHandler">
<lst name="defaults">

43
solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
View File

@ -17,13 +17,24 @@
package org.apache.solr.client.solrj.embedded;
import java.io.ByteArrayInputStream;
import java.util.Map;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.InputStreamEntity;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrExampleTests;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.util.ExternalPaths;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.noggit.ObjectBuilder;
/**
* TODO? perhaps use:
@ -52,4 +63,36 @@ public class SolrExampleJettyTest extends SolrExampleTests {
// expected
}
}
@Test
public void testArbitraryJsonIndexing() throws Exception {
HttpSolrServer server = (HttpSolrServer) getSolrServer();
server.deleteByQuery("*:*");
server.commit();
assertNumFound("*:*", 0); // make sure it got in
// two docs, one with uniqueKey, another without it
String json = "{\"id\":\"abc1\", \"name\": \"name1\"} {\"name\" : \"name2\"}";
HttpClient httpClient = server.getHttpClient();
HttpPost post = new HttpPost(server.getBaseURL() + "/update/json/docs");
post.setHeader("Content-Type", "application/json");
post.setEntity(new InputStreamEntity(new ByteArrayInputStream(json.getBytes("UTF-8")), -1));
HttpResponse response = httpClient.execute(post);
assertEquals(200, response.getStatusLine().getStatusCode());
server.commit();
QueryResponse rsp = getSolrServer().query(new SolrQuery("*:*"));
assertEquals(2,rsp.getResults().getNumFound());
SolrDocument doc = rsp.getResults().get(0);
String src = (String) doc.getFieldValue("_src");
Map m = (Map) ObjectBuilder.fromJSON(src);
assertEquals("abc1",m.get("id"));
assertEquals("name1",m.get("name"));
doc = rsp.getResults().get(1);
src = (String) doc.getFieldValue("_src");
m = (Map) ObjectBuilder.fromJSON(src);
assertEquals("name2",m.get("name"));
}
}