diff --git a/solr/contrib/dataimporthandler/CHANGES.txt b/solr/contrib/dataimporthandler/CHANGES.txt index 85acdcfcb96..c0ca6fc8f2e 100644 --- a/solr/contrib/dataimporthandler/CHANGES.txt +++ b/solr/contrib/dataimporthandler/CHANGES.txt @@ -18,6 +18,10 @@ New Features ---------------------- * SOLR-1499: Added SolrEntityProcessor that imports data from another Solr core or instance based on a specified query. (Lance Norskog, Erik Hatcher, Pulkit Singhal, Ahmet Arslan, Luca Cavanna, Martijn van Groningen) + Additional Work: + SOLR-3190: Minor improvements to SolrEntityProcessor. Add more consistency between solr parameters + and parameters used in SolrEntityProcessor and ability to specify a custom HttpClient instance. + (Luca Cavanna via Martijn van Groningen) Changes in Runtime Behavior ---------------------- diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java index c2c169e4e73..6a1b411c811 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java @@ -17,16 +17,6 @@ package org.apache.solr.handler.dataimport; * limitations under the License. */ -import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE; -import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.solr.client.solrj.SolrQuery; @@ -37,9 +27,20 @@ import org.apache.solr.client.solrj.impl.XMLResponseParser; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CommonParams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE; +import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow; + /** *

* An implementation of {@link EntityProcessor} which fetches values from a @@ -58,15 +59,8 @@ public class SolrEntityProcessor extends EntityProcessorBase { public static final String SOLR_SERVER = "url"; public static final String QUERY = "query"; - /** - * (format="javabin|xml") default is javabin - */ - public static final String FORMAT = "format"; - public static final String ROWS = "rows"; - public static final String FIELDS = "fields"; - public static final String FQ = "fq"; public static final String TIMEOUT = "timeout"; - + public static final int TIMEOUT_SECS = 5 * 60; // 5 minutes public static final int ROWS_DEFAULT = 50; @@ -75,10 +69,22 @@ public class SolrEntityProcessor extends EntityProcessorBase { private int rows = ROWS_DEFAULT; private String[] filterQueries; private String[] fields; + private String queryType; private int timeout = TIMEOUT_SECS; private boolean initDone = false; - + + /** + * Factory method that returns a {@link HttpClient} instance used for interfacing with a source Solr service. + * One can override this method to return a differently configured {@link HttpClient} instance. + * For example configure https and http authentication. + * + * @return a {@link HttpClient} instance used for interfacing with a source Solr service + */ + protected HttpClient getHttpClient() { + return new HttpClient(new MultiThreadedHttpConnectionManager()); + } + @Override protected void firstInit(Context context) { super.firstInit(context); @@ -89,23 +95,21 @@ public class SolrEntityProcessor extends EntityProcessorBase { throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "SolrEntityProcessor: parameter 'url' is required"); } - HttpClient client = new HttpClient( - new MultiThreadedHttpConnectionManager()); + + HttpClient client = getHttpClient(); URL url = new URL(serverPath); - - if ("xml".equals(context.getResolvedEntityAttribute(FORMAT))) { - solrServer = new CommonsHttpSolrServer(url, client, - new XMLResponseParser(), false); + // (wt="javabin|xml") default is javabin + if ("xml".equals(context.getResolvedEntityAttribute(CommonParams.WT))) { + solrServer = new CommonsHttpSolrServer(url, client, new XMLResponseParser(), false); LOG.info("using XMLResponseParser"); } else { solrServer = new CommonsHttpSolrServer(url, client); LOG.info("using BinaryResponseParser"); } - } catch (MalformedURLException e) { throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e); } - + this.queryString = context.getResolvedEntityAttribute(QUERY); if (this.queryString == null) { throw new DataImportHandlerException( @@ -114,21 +118,21 @@ public class SolrEntityProcessor extends EntityProcessorBase { ); } - String rowsP = context.getResolvedEntityAttribute(ROWS); + String rowsP = context.getResolvedEntityAttribute(CommonParams.ROWS); if (rowsP != null) { rows = Integer.parseInt(rowsP); } - String fqAsString = context.getResolvedEntityAttribute(FQ); + String fqAsString = context.getResolvedEntityAttribute(CommonParams.FQ); if (fqAsString != null) { this.filterQueries = fqAsString.split(","); } - String fieldsAsString = context.getResolvedEntityAttribute(FIELDS); + String fieldsAsString = context.getResolvedEntityAttribute(CommonParams.FL); if (fieldsAsString != null) { this.fields = fieldsAsString.split(","); } - + this.queryType = context.getResolvedEntityAttribute(CommonParams.QT); String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT); if (timeoutAsString != null) { this.timeout = Integer.parseInt(timeoutAsString); @@ -181,6 +185,7 @@ public class SolrEntityProcessor extends EntityProcessorBase { solrQuery.addField(field); } } + solrQuery.setQueryType(queryType); solrQuery.setFilterQueries(filterQueries); solrQuery.setTimeAllowed(timeout * 1000); diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java index dbd7df085f7..89b95c89402 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java @@ -145,7 +145,7 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe try { addDocumentsToSolr(SOLR_DOCS); - runFullImport(generateDIHConfig("query='*:*' rows='2' fields='id,desc' onError='skip'", jetty.getLocalPort())); + runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", jetty.getLocalPort())); } catch (Exception e) { LOG.error(e.getMessage(), e); fail(e.getMessage()); @@ -178,7 +178,7 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe try { addDocumentsToSolr(generateSolrDocuments(7)); - runFullImport(generateDIHConfig("query='*:*' fields='id' rows='2'", jetty.getLocalPort())); + runFullImport(generateDIHConfig("query='*:*' fl='id' rows='2'", jetty.getLocalPort())); } catch (Exception e) { LOG.error(e.getMessage(), e); fail(e.getMessage()); @@ -231,7 +231,7 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe assertQ(req("*:*"), "//result[@numFound='0']"); try { - runFullImport(generateDIHConfig("query='*:*' rows='2' fields='id,desc' onError='skip'", jetty.getLocalPort())); + runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", jetty.getLocalPort())); } catch (Exception e) { LOG.error(e.getMessage(), e); fail(e.getMessage()); @@ -244,7 +244,7 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe assertQ(req("*:*"), "//result[@numFound='0']"); try { - runFullImport(generateDIHConfig("query='bogus:3' rows='2' fields='id,desc' onError='abort'", jetty.getLocalPort())); + runFullImport(generateDIHConfig("query='bogus:3' rows='2' fl='id,desc' onError='abort'", jetty.getLocalPort())); } catch (Exception e) { LOG.error(e.getMessage(), e); fail(e.getMessage());