mirror of https://github.com/apache/lucene.git
SOLR-782 -- Refactored SolrWriter to make it a concrete class and removed wrappers over SolrInputDocument
git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@706542 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
fa5f2fd1db
commit
be8a3a53ff
|
@ -33,6 +33,10 @@ Bug Fixes
|
||||||
Documentation
|
Documentation
|
||||||
----------------------
|
----------------------
|
||||||
|
|
||||||
|
Other
|
||||||
|
----------------------
|
||||||
|
1. SOLR-782: Refactored SolrWriter to make it a concrete class and removed wrappers over SolrInputDocument
|
||||||
|
(Noble Paul via shalin)
|
||||||
|
|
||||||
================== Release 1.3.0 20080915 ==================
|
================== Release 1.3.0 20080915 ==================
|
||||||
|
|
||||||
|
|
|
@ -78,8 +78,6 @@ public class DataImportHandler extends RequestHandlerBase implements
|
||||||
|
|
||||||
private List<SolrInputDocument> debugDocuments;
|
private List<SolrInputDocument> debugDocuments;
|
||||||
|
|
||||||
private DebugLogger debugLogger;
|
|
||||||
|
|
||||||
private boolean debugEnabled = true;
|
private boolean debugEnabled = true;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -180,8 +178,7 @@ public class DataImportHandler extends RequestHandlerBase implements
|
||||||
req.getCore().getUpdateProcessingChain(params.get(UpdateParams.UPDATE_PROCESSOR));
|
req.getCore().getUpdateProcessingChain(params.get(UpdateParams.UPDATE_PROCESSOR));
|
||||||
UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp);
|
UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp);
|
||||||
SolrResourceLoader loader = req.getCore().getResourceLoader();
|
SolrResourceLoader loader = req.getCore().getResourceLoader();
|
||||||
SolrWriter sw = getSolrWriter(processor, loader, req
|
SolrWriter sw = getSolrWriter(processor, loader);
|
||||||
.getSchema());
|
|
||||||
|
|
||||||
if (requestParams.debug) {
|
if (requestParams.debug) {
|
||||||
if (debugEnabled) {
|
if (debugEnabled) {
|
||||||
|
@ -189,9 +186,8 @@ public class DataImportHandler extends RequestHandlerBase implements
|
||||||
importer.runCmd(requestParams, sw, variables);
|
importer.runCmd(requestParams, sw, variables);
|
||||||
rsp.add("mode", "debug");
|
rsp.add("mode", "debug");
|
||||||
rsp.add("documents", debugDocuments);
|
rsp.add("documents", debugDocuments);
|
||||||
if (debugLogger != null)
|
if (sw.debugLogger != null)
|
||||||
rsp.add("verbose-output", debugLogger.output);
|
rsp.add("verbose-output", sw.debugLogger.output);
|
||||||
debugLogger = null;
|
|
||||||
debugDocuments = null;
|
debugDocuments = null;
|
||||||
} else {
|
} else {
|
||||||
message = DataImporter.MSG.DEBUG_NOT_ENABLED;
|
message = DataImporter.MSG.DEBUG_NOT_ENABLED;
|
||||||
|
@ -266,14 +262,13 @@ public class DataImportHandler extends RequestHandlerBase implements
|
||||||
}
|
}
|
||||||
|
|
||||||
private SolrWriter getSolrWriter(final UpdateRequestProcessor processor,
|
private SolrWriter getSolrWriter(final UpdateRequestProcessor processor,
|
||||||
final SolrResourceLoader loader, final IndexSchema schema) {
|
final SolrResourceLoader loader) {
|
||||||
|
|
||||||
return new SolrWriter(processor, loader.getConfigDir()) {
|
return new SolrWriter(processor, loader.getConfigDir()) {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean upload(SolrDoc d) {
|
public boolean upload(SolrInputDocument document) {
|
||||||
try {
|
try {
|
||||||
SolrInputDocument document = ((SolrDocumentWrapper) d).doc;
|
|
||||||
if (requestParams.debug) {
|
if (requestParams.debug) {
|
||||||
if (debugDocuments == null)
|
if (debugDocuments == null)
|
||||||
debugDocuments = new ArrayList<SolrInputDocument>();
|
debugDocuments = new ArrayList<SolrInputDocument>();
|
||||||
|
@ -283,53 +278,15 @@ public class DataImportHandler extends RequestHandlerBase implements
|
||||||
importer.getDocBuilder().abort();
|
importer.getDocBuilder().abort();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return super.upload(document);
|
return super.upload(document);
|
||||||
} catch (RuntimeException e) {
|
} catch (RuntimeException e) {
|
||||||
LOG.error( "Exception while adding: " + d, e);
|
LOG.error( "Exception while adding: " + document, e);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void log(int event, String name, Object row) {
|
|
||||||
if (debugLogger == null) {
|
|
||||||
debugLogger = new DebugLogger();
|
|
||||||
}
|
|
||||||
debugLogger.log(event, name, row);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public SolrDoc getSolrDocInstance() {
|
|
||||||
return new SolrDocumentWrapper();
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
static class SolrDocumentWrapper implements SolrWriter.SolrDoc {
|
|
||||||
SolrInputDocument doc;
|
|
||||||
|
|
||||||
public SolrDocumentWrapper() {
|
|
||||||
doc = new SolrInputDocument();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDocumentBoost(float boost) {
|
|
||||||
doc.setDocumentBoost(boost);
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object getField(String field) {
|
|
||||||
return doc.getField(field);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void addField(String name, Object value, float boost) {
|
|
||||||
doc.addField(name, value, boost);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String toString() {
|
|
||||||
return doc.toString();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public NamedList getStatistics() {
|
public NamedList getStatistics() {
|
||||||
|
|
|
@ -18,6 +18,7 @@
|
||||||
package org.apache.solr.handler.dataimport;
|
package org.apache.solr.handler.dataimport;
|
||||||
|
|
||||||
import org.apache.solr.core.SolrCore;
|
import org.apache.solr.core.SolrCore;
|
||||||
|
import org.apache.solr.common.SolrInputDocument;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
@ -229,7 +230,7 @@ public class DocBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
private void buildDocument(VariableResolverImpl vr, SolrWriter.SolrDoc doc,
|
private void buildDocument(VariableResolverImpl vr, SolrInputDocument doc,
|
||||||
Map<String, Object> pk, DataConfig.Entity entity, boolean isRoot,
|
Map<String, Object> pk, DataConfig.Entity entity, boolean isRoot,
|
||||||
ContextImpl parentCtx) {
|
ContextImpl parentCtx) {
|
||||||
|
|
||||||
|
@ -272,7 +273,7 @@ public class DocBuilder {
|
||||||
ctx.getDocSession().clear();
|
ctx.getDocSession().clear();
|
||||||
else
|
else
|
||||||
ctx.setDocSession(new HashMap<String, Object>());
|
ctx.setDocSession(new HashMap<String, Object>());
|
||||||
doc = writer.getSolrDocInstance();
|
doc = new SolrInputDocument();
|
||||||
DataConfig.Entity e = entity;
|
DataConfig.Entity e = entity;
|
||||||
while (e.parentEntity != null) {
|
while (e.parentEntity != null) {
|
||||||
addFields(e.parentEntity, doc, (Map<String, Object>) vr
|
addFields(e.parentEntity, doc, (Map<String, Object>) vr
|
||||||
|
@ -351,7 +352,7 @@ public class DocBuilder {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void setDocumentBoost(SolrWriter.SolrDoc doc, Map<String, Object> arow) {
|
private void setDocumentBoost(SolrInputDocument doc, Map<String, Object> arow) {
|
||||||
Object v = arow.get(DOC_BOOST);
|
Object v = arow.get(DOC_BOOST);
|
||||||
float value = 1.0f;
|
float value = 1.0f;
|
||||||
if (v instanceof Number) {
|
if (v instanceof Number) {
|
||||||
|
@ -363,7 +364,7 @@ public class DocBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
private void addFields(DataConfig.Entity entity, SolrWriter.SolrDoc doc,
|
private void addFields(DataConfig.Entity entity, SolrInputDocument doc,
|
||||||
Map<String, Object> arow) {
|
Map<String, Object> arow) {
|
||||||
DataConfig.Entity parentMost = entity;
|
DataConfig.Entity parentMost = entity;
|
||||||
while (parentMost.parentEntity != null)
|
while (parentMost.parentEntity != null)
|
||||||
|
@ -383,7 +384,7 @@ public class DocBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addFieldValue(DataConfig.Field field, Map<String, Object> arow,
|
private void addFieldValue(DataConfig.Field field, Map<String, Object> arow,
|
||||||
Map<String, Object> lowerCaseMap, SolrWriter.SolrDoc doc) {
|
Map<String, Object> lowerCaseMap, SolrInputDocument doc) {
|
||||||
if (!field.toWrite)
|
if (!field.toWrite)
|
||||||
return;
|
return;
|
||||||
Object value = arow.get(field.column);
|
Object value = arow.get(field.column);
|
||||||
|
|
|
@ -16,18 +16,18 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.solr.handler.dataimport;
|
package org.apache.solr.handler.dataimport;
|
||||||
|
|
||||||
|
import org.apache.solr.common.SolrInputDocument;
|
||||||
import org.apache.solr.update.AddUpdateCommand;
|
import org.apache.solr.update.AddUpdateCommand;
|
||||||
import org.apache.solr.update.CommitUpdateCommand;
|
import org.apache.solr.update.CommitUpdateCommand;
|
||||||
import org.apache.solr.update.DeleteUpdateCommand;
|
import org.apache.solr.update.DeleteUpdateCommand;
|
||||||
import org.apache.solr.update.processor.UpdateRequestProcessor;
|
import org.apache.solr.update.processor.UpdateRequestProcessor;
|
||||||
import org.apache.solr.common.SolrInputDocument;
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
import java.text.ParseException;
|
import java.text.ParseException;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p>
|
* <p>
|
||||||
|
@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory;
|
||||||
* @version $Id$
|
* @version $Id$
|
||||||
* @since solr 1.3
|
* @since solr 1.3
|
||||||
*/
|
*/
|
||||||
public abstract class SolrWriter {
|
public class SolrWriter {
|
||||||
private static final Logger log = LoggerFactory.getLogger(SolrWriter.class);
|
private static final Logger log = LoggerFactory.getLogger(SolrWriter.class);
|
||||||
|
|
||||||
static final String IMPORTER_PROPERTIES = "dataimport.properties";
|
static final String IMPORTER_PROPERTIES = "dataimport.properties";
|
||||||
|
@ -51,6 +51,8 @@ public abstract class SolrWriter {
|
||||||
|
|
||||||
private final String configDir;
|
private final String configDir;
|
||||||
|
|
||||||
|
DebugLogger debugLogger;
|
||||||
|
|
||||||
public SolrWriter(UpdateRequestProcessor processor, String confDir) {
|
public SolrWriter(UpdateRequestProcessor processor, String confDir) {
|
||||||
this.processor = processor;
|
this.processor = processor;
|
||||||
configDir = confDir;
|
configDir = confDir;
|
||||||
|
@ -66,10 +68,10 @@ public abstract class SolrWriter {
|
||||||
command.overwriteCommitted = true;
|
command.overwriteCommitted = true;
|
||||||
processor.processAdd(command);
|
processor.processAdd(command);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
log.error( "Exception while adding: " + d, e);
|
log.error("Exception while adding: " + d, e);
|
||||||
return false;
|
return false;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.warn( "Error creating document : " + d, e);
|
log.warn("Error creating document : " + d, e);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -85,7 +87,7 @@ public abstract class SolrWriter {
|
||||||
delCmd.fromCommitted = true;
|
delCmd.fromCommitted = true;
|
||||||
processor.processDelete(delCmd);
|
processor.processDelete(delCmd);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
log.error( "Exception while deleteing: " + id, e);
|
log.error("Exception while deleteing: " + id, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -145,7 +147,7 @@ public abstract class SolrWriter {
|
||||||
props.load(propInput);
|
props.load(propInput);
|
||||||
log.info("Read " + SolrWriter.IMPORTER_PROPERTIES);
|
log.info("Read " + SolrWriter.IMPORTER_PROPERTIES);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.warn( "Unable to read: " + SolrWriter.IMPORTER_PROPERTIES);
|
log.warn("Unable to read: " + SolrWriter.IMPORTER_PROPERTIES);
|
||||||
} finally {
|
} finally {
|
||||||
try {
|
try {
|
||||||
if (propInput != null)
|
if (propInput != null)
|
||||||
|
@ -167,7 +169,7 @@ public abstract class SolrWriter {
|
||||||
delCmd.fromPending = true;
|
delCmd.fromPending = true;
|
||||||
processor.processDelete(delCmd);
|
processor.processDelete(delCmd);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
log.error( "Exception while deleting by query: " + query, e);
|
log.error("Exception while deleting by query: " + query, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -176,7 +178,7 @@ public abstract class SolrWriter {
|
||||||
CommitUpdateCommand commit = new CommitUpdateCommand(optimize);
|
CommitUpdateCommand commit = new CommitUpdateCommand(optimize);
|
||||||
processor.processCommit(commit);
|
processor.processCommit(commit);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error( "Exception while solr commit.", e);
|
log.error("Exception while solr commit.", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -240,18 +242,6 @@ public abstract class SolrWriter {
|
||||||
this.persistStartTime(date);
|
this.persistStartTime(date);
|
||||||
}
|
}
|
||||||
|
|
||||||
public abstract SolrDoc getSolrDocInstance();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* <p>
|
|
||||||
* Write the document to the index
|
|
||||||
* </p>
|
|
||||||
*
|
|
||||||
* @param d . The Document warapper object
|
|
||||||
* @return a boolean value denoting success (true) or failure (false)
|
|
||||||
*/
|
|
||||||
public abstract boolean upload(SolrDoc d);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This method is used for verbose debugging
|
* This method is used for verbose debugging
|
||||||
*
|
*
|
||||||
|
@ -259,20 +249,11 @@ public abstract class SolrWriter {
|
||||||
* @param name Name of the entity/transformer
|
* @param name Name of the entity/transformer
|
||||||
* @param row The actual data . Can be a Map<String,object> or a List<Map<String,object>>
|
* @param row The actual data . Can be a Map<String,object> or a List<Map<String,object>>
|
||||||
*/
|
*/
|
||||||
public abstract void log(int event, String name, Object row);
|
public void log(int event, String name, Object row) {
|
||||||
|
if (debugLogger == null) {
|
||||||
/**
|
debugLogger = new DebugLogger();
|
||||||
* The purpose of this interface to provide pluggable implementations for Solr
|
}
|
||||||
* 1.2 & 1.3 The implementation can choose to wrap appropriate Objects based
|
debugLogger.log(event, name, row);
|
||||||
* on the version
|
|
||||||
*/
|
|
||||||
public static interface SolrDoc {
|
|
||||||
|
|
||||||
public void addField(String name, Object value, float boost);
|
|
||||||
|
|
||||||
public Object getField(String field);
|
|
||||||
|
|
||||||
public void setDocumentBoost(float boost);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static final int START_ENTITY = 1, END_ENTITY = 2,
|
public static final int START_ENTITY = 1, END_ENTITY = 2,
|
||||||
|
|
|
@ -170,12 +170,8 @@ public class TestDocBuilder {
|
||||||
super(null, ".");
|
super(null, ".");
|
||||||
}
|
}
|
||||||
|
|
||||||
public SolrDoc getSolrDocInstance() {
|
public boolean upload(SolrInputDocument doc) {
|
||||||
return new DataImportHandler.SolrDocumentWrapper();
|
return docs.add(doc);
|
||||||
}
|
|
||||||
|
|
||||||
public boolean upload(SolrDoc d) {
|
|
||||||
return docs.add(((DataImportHandler.SolrDocumentWrapper) d).doc);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void log(int event, String name, Object row) {
|
public void log(int event, String name, Object row) {
|
||||||
|
|
Loading…
Reference in New Issue