From cf0a31f1f737c08953107fbc932a561866af20cc Mon Sep 17 00:00:00 2001
From: Shalin Shekhar Mangar <shalin@apache.org>
Date: Thu, 11 Dec 2008 09:05:39 +0000
Subject: [PATCH] SOLR-846 -- Reduce memory consumption during delta import by
 removing keys when used

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@725627 13f79535-47bb-0310-9956-ffa450edef68
---
 contrib/dataimporthandler/CHANGES.txt         |  6 ++++--
 .../solr/handler/dataimport/DocBuilder.java   | 19 +++++++++++++------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/contrib/dataimporthandler/CHANGES.txt b/contrib/dataimporthandler/CHANGES.txt
index b86beeb80d3..0c0a540cc4e 100644
--- a/contrib/dataimporthandler/CHANGES.txt
+++ b/contrib/dataimporthandler/CHANGES.txt
@@ -35,11 +35,13 @@ New Features
 5. SOLR-887:  A Transformer to strip HTML tags.
               (Ahmed Hammad via shalin)
 
-6. SOLR-886: DataImportHandler should rollback when an import fails or it is aborted (shalin)
+6. SOLR-886:  DataImportHandler should rollback when an import fails or it is aborted
+              (shalin)
 
 Optimizations
 ----------------------
-
+1. SOLR-846:  Reduce memory consumption during delta import by removing keys when used
+              (Ricky Leung, Noble Paul via shalin)
 
 Bug Fixes
 ----------------------
diff --git a/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java b/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
index 8bbfc09da37..ad943f59235 100644
--- a/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
+++ b/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
@@ -207,12 +207,16 @@ public class DocBuilder {
       // Make sure that documents are not re-created
       allPks.removeAll(deletedKeys);
     }
+    deletedKeys = null;
 
     statusMessages.put("Total Changed Documents", allPks.size());
-    for (Map<String, Object> pk : allPks) {
-      VariableResolverImpl vri = getVariableResolver(dataImporter);
-      vri.addNamespace(DataConfig.IMPORTER_NS + ".delta", pk);
-      buildDocument(vri, null, pk, root, true, null);
+    VariableResolverImpl vri = getVariableResolver(dataImporter);
+    Iterator<Map<String, Object>> pkIter = allPks.iterator();
+    while (pkIter.hasNext()) {
+      Map<String, Object> map = pkIter.next();
+      vri.addNamespace(DataConfig.IMPORTER_NS + ".delta", map);
+      buildDocument(vri, null, map, root, true, null);
+      pkIter.remove();
     }
 
     if (!stop.get()) {
@@ -223,8 +227,11 @@ public class DocBuilder {
 
   private void deleteAll(Set<Map<String, Object>> deletedKeys) {
     LOG.info("Deleting stale documents ");
-    for (Map<String, Object> deletedKey : deletedKeys) {
-      writer.deleteDoc(deletedKey.get(root.pk));
+    Iterator<Map<String, Object>> iter = deletedKeys.iterator();
+    while (iter.hasNext()) {
+      Map<String, Object> map = iter.next();
+      writer.deleteDoc(map.get(root.pk));
+      iter.remove();
     }
   }