Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-782 followup -- Removed multiple document nodes in the configuration xml. Removed support for 'default' variables, they are automatically available as request parameters. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@730055 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shalin Shekhar Mangar 2008-12-30 06:42:48 +00:00
parent fd6b300d8b
commit 9f6e8ea8ec
8 changed files with 57 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
contrib/dataimporthandler/CHANGES.txt
View File

@ -93,7 +93,9 @@ Documentation
Other
----------------------
1. SOLR-782: Refactored SolrWriter to make it a concrete class and removed wrappers over SolrInputDocument
1. SOLR-782: Refactored SolrWriter to make it a concrete class and removed wrappers over SolrInputDocument.
Refactored to load Evaluators lazily. Removed multiple document nodes in the configuration xml.
Removed support for 'default' variables, they are automatically available as request parameters.
(Noble Paul via shalin)
================== Release 1.3.0 20080915 ==================

33
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
View File

@ -41,9 +41,7 @@ import java.util.*;
* @since solr 1.3
*/
public class DataConfig {
public List<Document> documents;
private Map<String, Document> documentCache;
public Document document;
public List<Map<String, String >> functions = new ArrayList<Map<String ,String>>();
@ -53,20 +51,7 @@ public class DataConfig {
public Map<String, SchemaField> lowerNameVsSchemaField = new HashMap<String, SchemaField>();
public Document getDocumentByName(String name) {
if (documentCache == null) {
documentCache = new HashMap<String, Document>();
for (Document document : documents)
documentCache.put(document.name, document);
}
return documentCache.get(name);
}
public static class Document {
// TODO - remove this
public String name;
// TODO - remove from here and add it to entity
public String deleteQuery;
@ -76,7 +61,6 @@ public class DataConfig {
}
public Document(Element element) {
this.name = getStringAttribute(element, NAME, null);
this.deleteQuery = getStringAttribute(element, "deleteQuery", null);
List<Element> l = getChildNodes(element, "entity");
for (Element e : l)
@ -222,10 +206,11 @@ public class DataConfig {
public void readFromXml(Element e) {
List<Element> n = getChildNodes(e, "document");
if (!n.isEmpty())
documents = new ArrayList<Document>();
for (Element element : n)
documents.add(new Document(element));
if (n.isEmpty()) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "DataImportHandler " +
"configuration file must have one <document> node.");
}
document = new Document(n.get(0));
n = getChildNodes(e, SCRIPT);
if (!n.isEmpty()) {
@ -307,10 +292,8 @@ public class DataConfig {
}
public void clearCaches() {
for (Document document : documents)
for (Entity entity : document.entities)
entity.clearCache();
for (Entity entity : document.entities)
entity.clearCache();
}
public static final String SCRIPT = "script";

9
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
View File

@ -161,7 +161,7 @@ public class DataImportHandler extends RequestHandlerBase implements
}
if (command != null && DataImporter.ABORT_CMD.equals(command)) {
importer.runCmd(requestParams, null, null);
importer.runCmd(requestParams, null);
} else if (importer.isBusy()) {
message = DataImporter.MSG.CMD_RUNNING;
} else if (command != null) {
@ -177,7 +177,7 @@ public class DataImportHandler extends RequestHandlerBase implements
if (requestParams.debug) {
if (debugEnabled) {
// Synchronous request for the debug mode
importer.runCmd(requestParams, sw, variables);
importer.runCmd(requestParams, sw);
rsp.add("mode", "debug");
rsp.add("documents", debugDocuments);
if (sw.debugLogger != null)
@ -188,7 +188,7 @@ public class DataImportHandler extends RequestHandlerBase implements
}
} else {
// Asynchronous request for normal mode
importer.runAsync(requestParams, sw, variables);
importer.runAsync(requestParams, sw);
}
} else if (DataImporter.RELOAD_CONF_CMD.equals(command)) {
importer = null;
@ -222,8 +222,7 @@ public class DataImportHandler extends RequestHandlerBase implements
@SuppressWarnings("unchecked")
private void processConfiguration(NamedList defaults) {
if (defaults == null) {
LOG
.info("No configuration specified in solrconfig.xml for DataImportHandler");
LOG.info("No configuration specified in solrconfig.xml for DataImportHandler");
return;
}

46
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
View File

@ -95,15 +95,13 @@ public class DataImporter {
config.lowerNameVsSchemaField.put(entry.getKey().toLowerCase(), entry.getValue());
}
for (DataConfig.Document document : config.documents) {
for (DataConfig.Entity e : document.entities) {
Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
initEntity(e, fields, false);
String errs = verifyWithSchema(fields);
if (errs != null) {
throw new DataImportHandlerException(
DataImportHandlerException.SEVERE, errs);
}
for (DataConfig.Entity e : config.document.entities) {
Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
initEntity(e, fields, false);
String errs = verifyWithSchema(fields);
if (errs != null) {
throw new DataImportHandlerException(
DataImportHandlerException.SEVERE, errs);
}
}
}
@ -150,8 +148,9 @@ public class DataImporter {
void loadAndInit(String configStr) {
loadDataConfig(configStr);
Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
DataConfig.Entity e = getConfig().documents.get(0).entities.get(0);
initEntity(e, fields, false);
for (DataConfig.Entity entity : config.document.entities) {
initEntity(entity, fields, false);
}
}
void loadDataConfig(String configFile) {
@ -308,8 +307,7 @@ public class DataImporter {
return importLock.isLocked();
}
public void doFullImport(SolrWriter writer, RequestParams requestParams,
Map<String, String> variables) {
public void doFullImport(SolrWriter writer, RequestParams requestParams) {
LOG.info("Starting Full Import");
setStatus(Status.RUNNING_FULL_DUMP);
@ -319,8 +317,8 @@ public class DataImporter {
try {
if (requestParams.clean)
writer.doDeleteAll();
docBuilder = new DocBuilder(this, writer, requestParams, variables);
docBuilder.execute(getConfig().documents.get(0).name);
docBuilder = new DocBuilder(this, writer, requestParams);
docBuilder.execute();
if (!requestParams.debug)
cumulativeStatistics.add(docBuilder.importStatistics);
} catch (Throwable t) {
@ -334,8 +332,7 @@ public class DataImporter {
}
public void doDeltaImport(SolrWriter writer, RequestParams requestParams,
Map<String, String> variables) {
public void doDeltaImport(SolrWriter writer, RequestParams requestParams) {
LOG.info("Starting Delta Import");
setStatus(Status.RUNNING_DELTA_DUMP);
@ -343,8 +340,8 @@ public class DataImporter {
if (requestParams.commit) {
setIndexStartTime(new Date());
}
docBuilder = new DocBuilder(this, writer, requestParams, variables);
docBuilder.execute(config.documents.get(0).name);
docBuilder = new DocBuilder(this, writer, requestParams);
docBuilder.execute();
if (!requestParams.debug)
cumulativeStatistics.add(docBuilder.importStatistics);
} catch (Throwable t) {
@ -358,17 +355,16 @@ public class DataImporter {
}
public void runAsync(final RequestParams reqParams, final SolrWriter sw,
final Map<String, String> variables) {
public void runAsync(final RequestParams reqParams, final SolrWriter sw) {
new Thread() {
@Override
public void run() {
runCmd(reqParams, sw, variables);
runCmd(reqParams, sw);
}
}.start();
}
void runCmd(RequestParams reqParams, SolrWriter sw, Map<String, String> variables) {
void runCmd(RequestParams reqParams, SolrWriter sw) {
String command = reqParams.command;
if (command.equals(ABORT_CMD)) {
if (docBuilder != null) {
@ -382,9 +378,9 @@ public class DataImporter {
Date lastModified = sw.loadIndexStartTime();
setLastIndexTime(lastModified);
if (command.equals("full-import")) {
doFullImport(sw, reqParams, variables);
doFullImport(sw, reqParams);
} else if (command.equals(DELTA_IMPORT_CMD)) {
doDeltaImport(sw, reqParams, variables);
doDeltaImport(sw, reqParams);
}
} finally {
importLock.unlock();

33
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
View File

@ -59,50 +59,43 @@ public class DocBuilder {
boolean verboseDebug = false;
private Map<String, String> defaultVariables;
private Map<String, Object> session = new HashMap<String, Object>();
static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
public DocBuilder(DataImporter context, SolrWriter writer,
DataImporter.RequestParams reqParams, Map<String, String> variables) {
public DocBuilder(DataImporter context, SolrWriter writer, DataImporter.RequestParams reqParams) {
INSTANCE.set(this);
this.dataImporter = context;
this.writer = writer;
DataImporter.QUERY_COUNT.set(importStatistics.queryCount);
requestParameters = reqParams;
verboseDebug = requestParameters.debug && requestParameters.verbose;
defaultVariables = Collections.unmodifiableMap(variables);
}
public VariableResolverImpl getVariableResolver(DataImporter context) {
public VariableResolverImpl getVariableResolver() {
VariableResolverImpl resolver = new VariableResolverImpl();
Map<String, Object> indexerNamespace = new HashMap<String, Object>();
if (context.getLastIndexTime() != null)
if (dataImporter.getLastIndexTime() != null)
indexerNamespace.put(LAST_INDEX_TIME, DataImporter.DATE_TIME_FORMAT
.format(context.getLastIndexTime()));
indexerNamespace.put(INDEX_START_TIME, context.getIndexStartTime());
.format(dataImporter.getLastIndexTime()));
indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
indexerNamespace.put("request", requestParameters.requestParams);
indexerNamespace.put("defaults", defaultVariables);
indexerNamespace.put("functions", EvaluatorBag.getFunctionsNamespace(resolver,
dataImporter.getConfig().functions, this));
if (context.getConfig().script != null) {
if (dataImporter.getConfig().script != null) {
indexerNamespace
.put(DataConfig.SCRIPT, context.getConfig().script.script);
.put(DataConfig.SCRIPT, dataImporter.getConfig().script.script);
indexerNamespace.put(DataConfig.SCRIPT_LANG,
context.getConfig().script.language);
dataImporter.getConfig().script.language);
}
resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
return resolver;
}
@SuppressWarnings("unchecked")
public void execute(String docName) {
public void execute() {
dataImporter.store(DataImporter.STATUS_MSGS, statusMessages);
document = dataImporter.getConfig().getDocumentByName(docName);
if (document == null)
return;
document = dataImporter.getConfig().document;
final AtomicLong startTime = new AtomicLong(System.currentTimeMillis());
statusMessages.put(TIME_ELAPSED, new Object() {
public String toString() {
@ -180,14 +173,14 @@ public class DocBuilder {
@SuppressWarnings("unchecked")
private void doFullDump() {
addStatusMessage("Full Dump Started");
buildDocument(getVariableResolver(dataImporter), null, null, root, true,
buildDocument(getVariableResolver(), null, null, root, true,
null);
}
@SuppressWarnings("unchecked")
private void doDelta() {
addStatusMessage("Delta Dump started");
VariableResolverImpl resolver = getVariableResolver(dataImporter);
VariableResolverImpl resolver = getVariableResolver();
if (document.deleteQuery != null) {
writer.deleteByQuery(document.deleteQuery);
@ -211,7 +204,7 @@ public class DocBuilder {
deletedKeys = null;
statusMessages.put("Total Changed Documents", allPks.size());
VariableResolverImpl vri = getVariableResolver(dataImporter);
VariableResolverImpl vri = getVariableResolver();
Iterator<Map<String, Object>> pkIter = allPks.iterator();
while (pkIter.hasNext()) {
Map<String, Object> map = pkIter.next();

3
contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
View File

@ -75,8 +75,7 @@ public class TestDataConfig extends AbstractDataImportHandlerTest {
DataConfig dc = new DataConfig();
dc.readFromXml(doc.getDocumentElement());
Assert.assertEquals("atrimlisting",
dc.documents.get(0).entities.get(0).name);
Assert.assertEquals("atrimlisting", dc.document.entities.get(0).name);
}
private static final String xml = "<dataConfig>\n"

12
contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
View File

@ -48,14 +48,14 @@ public class TestDocBuilder {
DataImporter di = new DataImporter();
di.loadDataConfig(dc_singleEntity);
DataConfig cfg = di.getConfig();
DataConfig.Entity ent = cfg.documents.get(0).entities.get(0);
DataConfig.Entity ent = cfg.document.entities.get(0);
MockDataSource.setIterator("select * from x", new ArrayList().iterator());
ent.dataSrc = new MockDataSource();
ent.isDocRoot = true;
DataImporter.RequestParams rp = new DataImporter.RequestParams();
rp.command = "full-import";
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi, Collections.EMPTY_MAP);
di.runCmd(rp, swi);
Assert.assertEquals(Boolean.TRUE, swi.deleteAllCalled);
Assert.assertEquals(Boolean.TRUE, swi.commitCalled);
Assert.assertEquals(0, swi.docs.size());
@ -76,7 +76,7 @@ public class TestDocBuilder {
DataImporter di = new DataImporter();
di.loadDataConfig(dc_singleEntity);
DataConfig cfg = di.getConfig();
DataConfig.Entity ent = cfg.documents.get(0).entities.get(0);
DataConfig.Entity ent = cfg.document.entities.get(0);
List l = new ArrayList();
l.add(createMap("id", 1, "desc", "one"));
MockDataSource.setIterator("select * from x", l.iterator());
@ -85,7 +85,7 @@ public class TestDocBuilder {
DataImporter.RequestParams rp = new DataImporter.RequestParams();
rp.command = "full-import";
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi, Collections.EMPTY_MAP);
di.runCmd(rp, swi);
Assert.assertEquals(Boolean.TRUE, swi.deleteAllCalled);
Assert.assertEquals(Boolean.TRUE, swi.commitCalled);
Assert.assertEquals(1, swi.docs.size());
@ -115,7 +115,7 @@ public class TestDocBuilder {
DataImporter di = new DataImporter();
di.loadDataConfig(dc_singleEntity);
DataConfig cfg = di.getConfig();
DataConfig.Entity ent = cfg.documents.get(0).entities.get(0);
DataConfig.Entity ent = cfg.document.entities.get(0);
ent.isDocRoot = true;
DataImporter.RequestParams rp = new DataImporter.RequestParams();
rp.command = "full-import";
@ -127,7 +127,7 @@ public class TestDocBuilder {
MockDataSource.setIterator("select * from x", l.iterator());
ent.dataSrc = new MockDataSource();
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi, Collections.EMPTY_MAP);
di.runCmd(rp, swi);
Assert.assertEquals(Boolean.TRUE, swi.deleteAllCalled);
Assert.assertEquals(Boolean.TRUE, swi.commitCalled);
Assert.assertEquals(3, swi.docs.size());

2
contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFieldReader.java
View File

@ -43,7 +43,7 @@ public class TestFieldReader {
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(createMap("xml", xml));
MockDataSource.setIterator("select * from a", l.iterator());
di.runCmd(rp, sw, new HashMap<String, String>());
di.runCmd(rp, sw);
Assert.assertEquals(sw.docs.get(0).getFieldValue("y"), "Hello");
MockDataSource.clearCache();
}