Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-3671: DIHWriter fix 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1592583 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
James Dyer 2014-05-05 17:40:32 +00:00
parent 88846a958d
commit 3c14dc6a57
5 changed files with 133 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -158,6 +158,8 @@ Other Changes
* SOLR-6022: Deprecate getAnalyzer() in IndexField and FieldType, and add getIndexAnalyzer().
(Ryan Ernst)
* SOLR_3671: Fix DIHWriter interface usage so users may implement writers that output
documents to a location external to Solr (ex. a NoSql db). (Roman Chyla via James Dyer)
Build
---------------------

57
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
View File

@ -17,6 +17,7 @@
package org.apache.solr.handler.dataimport;
import static org.apache.solr.handler.dataimport.DataImporter.IMPORT_CMD;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CommonParams;
@ -43,6 +44,8 @@ import org.apache.solr.util.plugin.SolrCoreAware;
import java.util.*;
import java.io.StringReader;
import java.lang.reflect.Constructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
@ -76,6 +79,9 @@ public class DataImportHandler extends RequestHandlerBase implements
private String myName = "dataimport";
private static final String PARAM_WRITER_IMPL = "writerImpl";
private static final String DEFAULT_WRITER_NAME = "SolrWriter";
@Override
@SuppressWarnings("unchecked")
public void init(NamedList args) {
@ -171,7 +177,7 @@ public class DataImportHandler extends RequestHandlerBase implements
req.getCore().getUpdateProcessingChain(params.get(UpdateParams.UPDATE_CHAIN));
UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp);
SolrResourceLoader loader = req.getCore().getResourceLoader();
SolrWriter sw = getSolrWriter(processor, loader, requestParams, req);
DIHWriter sw = getSolrWriter(processor, loader, requestParams, req);
if (requestParams.isDebug()) {
if (debugEnabled) {
@ -224,21 +230,42 @@ public class DataImportHandler extends RequestHandlerBase implements
return result;
}
private SolrWriter getSolrWriter(final UpdateRequestProcessor processor,
final SolrResourceLoader loader, final RequestInfo requestParams, SolrQueryRequest req) {
return new SolrWriter(processor, req) {
@Override
public boolean upload(SolrInputDocument document) {
try {
return super.upload(document);
} catch (RuntimeException e) {
LOG.error( "Exception while adding: " + document, e);
return false;
}
private DIHWriter getSolrWriter(final UpdateRequestProcessor processor,
final SolrResourceLoader loader, final RequestInfo requestParams,
SolrQueryRequest req) {
SolrParams reqParams = req.getParams();
String writerClassStr = null;
if (reqParams != null && reqParams.get(PARAM_WRITER_IMPL) != null) {
writerClassStr = (String) reqParams.get(PARAM_WRITER_IMPL);
}
DIHWriter writer;
if (writerClassStr != null
&& !writerClassStr.equals(DEFAULT_WRITER_NAME)
&& !writerClassStr.equals(DocBuilder.class.getPackage().getName() + "."
+ DEFAULT_WRITER_NAME)) {
try {
@SuppressWarnings("unchecked")
Class<DIHWriter> writerClass = DocBuilder.loadClass(writerClassStr, req.getCore());
Constructor<DIHWriter> cnstr = writerClass.getConstructor(new Class[] {
UpdateRequestProcessor.class, SolrQueryRequest.class});
return cnstr.newInstance((Object) processor, (Object) req);
} catch (Exception e) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
"Unable to load Writer implementation:" + writerClassStr, e);
}
};
} else {
return new SolrWriter(processor, req) {
@Override
public boolean upload(SolrInputDocument document) {
try {
return super.upload(document);
} catch (RuntimeException e) {
LOG.error("Exception while adding: " + document, e);
return false;
}
}
};
}
}
@Override

10
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
View File

@ -399,7 +399,7 @@ public class DataImporter {
return importLock.isLocked();
}
public void doFullImport(SolrWriter writer, RequestInfo requestParams) {
public void doFullImport(DIHWriter writer, RequestInfo requestParams) {
LOG.info("Starting Full Import");
setStatus(Status.RUNNING_FULL_DUMP);
boolean success = false;
@ -425,14 +425,14 @@ public class DataImporter {
}
private void checkWritablePersistFile(SolrWriter writer, DIHProperties dihPropWriter) {
private void checkWritablePersistFile(DIHWriter writer, DIHProperties dihPropWriter) {
if (isDeltaImportSupported && !dihPropWriter.isWritable()) {
throw new DataImportHandlerException(SEVERE,
"Properties is not writable. Delta imports are supported by data config but will not work.");
}
}
public void doDeltaImport(SolrWriter writer, RequestInfo requestParams) {
public void doDeltaImport(DIHWriter writer, RequestInfo requestParams) {
LOG.info("Starting Delta Import");
setStatus(Status.RUNNING_DELTA_DUMP);
boolean success = false;
@ -457,7 +457,7 @@ public class DataImporter {
}
public void runAsync(final RequestInfo reqParams, final SolrWriter sw) {
public void runAsync(final RequestInfo reqParams, final DIHWriter sw) {
new Thread() {
@Override
public void run() {
@ -466,7 +466,7 @@ public class DataImporter {
}.start();
}
void runCmd(RequestInfo reqParams, SolrWriter sw) {
void runCmd(RequestInfo reqParams, DIHWriter sw) {
String command = reqParams.getCommand();
if (command.equals(ABORT_CMD)) {
if (docBuilder != null) {

23
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
View File

@ -80,13 +80,10 @@ public class DocBuilder {
private Map<String, Object> persistedProperties;
private DIHProperties propWriter;
private static final String PARAM_WRITER_IMPL = "writerImpl";
private static final String DEFAULT_WRITER_NAME = "SolrWriter";
private DebugLogger debugLogger;
private final RequestInfo reqParams;
@SuppressWarnings("unchecked")
public DocBuilder(DataImporter dataImporter, SolrWriter solrWriter, DIHProperties propWriter, RequestInfo reqParams) {
public DocBuilder(DataImporter dataImporter, DIHWriter solrWriter, DIHProperties propWriter, RequestInfo reqParams) {
INSTANCE.set(this);
this.dataImporter = dataImporter;
this.reqParams = reqParams;
@ -95,20 +92,7 @@ public class DocBuilder {
verboseDebug = reqParams.isDebug() && reqParams.getDebugInfo().verbose;
persistedProperties = propWriter.readIndexerProperties();
String writerClassStr = null;
if(reqParams!=null && reqParams.getRawParams() != null) {
writerClassStr = (String) reqParams.getRawParams().get(PARAM_WRITER_IMPL);
}
if(writerClassStr != null && !writerClassStr.equals(DEFAULT_WRITER_NAME) && !writerClassStr.equals(DocBuilder.class.getPackage().getName() + "." + DEFAULT_WRITER_NAME)) {
try {
Class<DIHWriter> writerClass = loadClass(writerClassStr, dataImporter.getCore());
this.writer = writerClass.newInstance();
} catch (Exception e) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Unable to load Writer implementation:" + writerClassStr, e);
}
} else {
writer = solrWriter;
}
writer = solrWriter;
ContextImpl ctx = new ContextImpl(null, null, null, null, reqParams.getRawParams(), null, this);
writer.init(ctx);
}
@ -905,9 +889,6 @@ public class DocBuilder {
return reqParams;
}
@SuppressWarnings("unchecked")
static Class loadClass(String name, SolrCore core) throws ClassNotFoundException {
try {

81
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestWriterImpl.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.*;
/**
* <p>
* Test for writerImpl paramater (to provide own SolrWriter)
* </p>
*
*
* @since solr 4.0
*/
public class TestWriterImpl extends AbstractDataImportHandlerTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-nodatasource-solrconfig.xml", "dataimport-schema.xml");
}
@Test
@SuppressWarnings("unchecked")
public void testDataConfigWithDataSource() throws Exception {
List rows = new ArrayList();
rows.add(createMap("id", "1", "desc", "one"));
rows.add(createMap("id", "2", "desc", "two"));
rows.add(createMap("id", "3", "desc", "break"));
rows.add(createMap("id", "4", "desc", "four"));
MockDataSource.setIterator("select * from x", rows.iterator());
Map extraParams = createMap("writerImpl", TestSolrWriter.class.getName(),
"commit", "true");
runFullImport(loadDataConfig("data-config-with-datasource.xml"),
extraParams);
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("id:3"), "//*[@numFound='0']");
assertQ(req("id:4"), "//*[@numFound='1']");
}
public static class TestSolrWriter extends SolrWriter {
public TestSolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
super(processor, req);
}
@Override
public boolean upload(SolrInputDocument doc) {
if (doc.getField("desc").getFirstValue().equals("break")) {
return false;
}
return super.upload(doc);
}
}
}