Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-801 -- Add support for configurable pre-import and post-import delete query per root-entity 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@737555 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shalin Shekhar Mangar 2009-01-25 18:05:41 +00:00
parent b822a6db8b
commit 4ee827471d
3 changed files with 34 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
contrib/dataimporthandler/CHANGES.txt
View File

@ -50,6 +50,9 @@ New Features
10.SOLR-938: Add event listener API for import start and end.
(Kay Kay, Noble Paul via shalin)
11.SOLR-801: Add support for configurable pre-import and post-import delete query per root-entity.
(Noble Paul via shalin)
Optimizations
----------------------
1. SOLR-846: Reduce memory consumption during delta import by removing keys when used

5
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
View File

@ -315,8 +315,6 @@ public class DataImporter {
setIndexStartTime(new Date());
try {
if (requestParams.clean)
writer.doDeleteAll();
docBuilder = new DocBuilder(this, writer, requestParams);
docBuilder.execute();
if (!requestParams.debug)
@ -481,6 +479,9 @@ public class DataImporter {
verbose = "true".equals(requestParams.get("verbose"))
|| "on".equals(requestParams.get("verbose"));
}
if (DELTA_IMPORT_CMD.equals(command)) {
clean = false;
}
if (requestParams.containsKey("commit"))
commit = Boolean.parseBoolean((String) requestParams.get("commit"));
if (requestParams.containsKey("start"))

30
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
View File

@ -133,18 +133,32 @@ public class DocBuilder {
if (document.onImportStart != null) {
invokeEventListener(document.onImportStart);
}
AtomicBoolean fullCleanDone = new AtomicBoolean(false);
//we must not do a delete of *:* multiple times if there are multiple root entities to be run
for (DataConfig.Entity e : document.entities) {
if (entities != null && !entities.contains(e.name))
continue;
root = e;
String delQuery = e.allAttributes.get("preImportDeleteQuery");
if(delQuery == null) delQuery="*:*";
if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP
&& dataImporter.getLastIndexTime() != null) {
cleanByQuery(delQuery , fullCleanDone);
doDelta();
delQuery = e.allAttributes.get("postImportDeleteQuery");
if(delQuery != null) {
fullCleanDone.set(false);
cleanByQuery(delQuery, fullCleanDone);
}
} else {
cleanByQuery(delQuery, fullCleanDone);
doFullDump();
delQuery = e.allAttributes.get("postImportDeleteQuery");
if(delQuery != null) {
fullCleanDone.set(false);
cleanByQuery(delQuery, fullCleanDone);
}
}
statusMessages.remove(DataImporter.MSG.TOTAL_DOC_PROCESSED);
}
@ -637,6 +651,18 @@ public class DocBuilder {
}
}
private void cleanByQuery(String delQuery, AtomicBoolean completeCleanDone) {
delQuery = getVariableResolver().replaceTokens(delQuery);
if (requestParameters.clean) {
if (delQuery == null && !completeCleanDone.get()) {
writer.deleteByQuery("*:*");
completeCleanDone.set(true);
} else {
writer.deleteByQuery(delQuery);
}
}
}
public static final String LAST_INDEX_TIME = "last_index_time";
public static final String INDEX_START_TIME = "index_start_time";
}