Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-846 -- Reduce memory consumption during delta import by removing keys when used 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@725627 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shalin Shekhar Mangar 2008-12-11 09:05:39 +00:00
parent 7549d04c32
commit cf0a31f1f7
2 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
contrib/dataimporthandler/CHANGES.txt
View File

@ -35,11 +35,13 @@ New Features
5. SOLR-887: A Transformer to strip HTML tags.
(Ahmed Hammad via shalin)
6. SOLR-886: DataImportHandler should rollback when an import fails or it is aborted (shalin)
6. SOLR-886: DataImportHandler should rollback when an import fails or it is aborted
(shalin)
Optimizations
----------------------
1. SOLR-846: Reduce memory consumption during delta import by removing keys when used
(Ricky Leung, Noble Paul via shalin)
Bug Fixes
----------------------

19
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
View File

@ -207,12 +207,16 @@ public class DocBuilder {
// Make sure that documents are not re-created
allPks.removeAll(deletedKeys);
}
deletedKeys = null;
statusMessages.put("Total Changed Documents", allPks.size());
for (Map<String, Object> pk : allPks) {
VariableResolverImpl vri = getVariableResolver(dataImporter);
vri.addNamespace(DataConfig.IMPORTER_NS + ".delta", pk);
buildDocument(vri, null, pk, root, true, null);
VariableResolverImpl vri = getVariableResolver(dataImporter);
Iterator<Map<String, Object>> pkIter = allPks.iterator();
while (pkIter.hasNext()) {
Map<String, Object> map = pkIter.next();
vri.addNamespace(DataConfig.IMPORTER_NS + ".delta", map);
buildDocument(vri, null, map, root, true, null);
pkIter.remove();
}
if (!stop.get()) {
@ -223,8 +227,11 @@ public class DocBuilder {
private void deleteAll(Set<Map<String, Object>> deletedKeys) {
LOG.info("Deleting stale documents ");
for (Map<String, Object> deletedKey : deletedKeys) {
writer.deleteDoc(deletedKey.get(root.pk));
Iterator<Map<String, Object>> iter = deletedKeys.iterator();
while (iter.hasNext()) {
Map<String, Object> map = iter.next();
writer.deleteDoc(map.get(root.pk));
iter.remove();
}
}