From 3e53e6561e50df81881e2e59512db7d7fbbe8ff0 Mon Sep 17 00:00:00 2001
From: Michael McCandless <mikemccand@apache.org>
Date: Mon, 17 May 2010 23:12:34 +0000
Subject: [PATCH] LUCENE-2467: fix memory leaks in IW when indexing very large
 docs

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@945420 13f79535-47bb-0310-9956-ffa450edef68
---
 lucene/CHANGES.txt                                |  3 +++
 .../org/apache/lucene/index/DocumentsWriter.java  | 15 +++++++++++----
 .../java/org/apache/lucene/index/IndexWriter.java |  6 +++++-
 .../apache/lucene/index/ReusableStringReader.java |  2 ++
 .../lucene/index/TermVectorsTermsWriter.java      |  2 +-
 5 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c40cf16fb22..4942b7f40d8 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -391,6 +391,9 @@ Bug fixes
   addition, all of IndexComnmit methods which threw 
   UnsupportedOperationException are now abstract. (Shai Erera)
 
+* LUCENE-2467: Fixed memory leaks in IndexWriter when large documents
+  are indexed.  (Mike McCandless)
+
 New features
 
 * LUCENE-2128: Parallelized fetching document frequencies during weight
diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
index 25a4f71e8c8..d8ce62793bd 100644
--- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
+++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
@@ -1314,11 +1314,14 @@ final class DocumentsWriter {
     }
 
     /* Return byte[]'s to the pool */
+
     @Override
     void recycleByteBlocks(byte[][] blocks, int start, int end) {
       synchronized(DocumentsWriter.this) {
-        for(int i=start;i<end;i++)
+        for(int i=start;i<end;i++) {
           freeByteBlocks.add(blocks[i]);
+          blocks[i] = null;
+        }
       }
     }
 
@@ -1326,9 +1329,11 @@ final class DocumentsWriter {
     void recycleByteBlocks(List<byte[]> blocks) {
       synchronized(DocumentsWriter.this) {
         final int size = blocks.size();
-        for(int i=0;i<size;i++)
+        for(int i=0;i<size;i++) {
           freeByteBlocks.add(blocks.get(i));
-  }
+          blocks.set(i, null);
+        }
+      }
     }
   }
 
@@ -1358,8 +1363,10 @@ final class DocumentsWriter {
 
   /* Return int[]s to the pool */
   synchronized void recycleIntBlocks(int[][] blocks, int start, int end) {
-    for(int i=start;i<end;i++)
+    for(int i=start;i<end;i++) {
       freeIntBlocks.add(blocks[i]);
+      blocks[i] = null;
+    }
   }
 
   ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator(BYTE_BLOCK_SIZE);
diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java
index 8147c9286cf..a3baa2e2b37 100644
--- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java
@@ -3624,7 +3624,11 @@ public class IndexWriter implements Closeable {
   // even while a flush is happening
   private synchronized final boolean doFlush(boolean flushDocStores, boolean flushDeletes) throws CorruptIndexException, IOException {
     try {
-      return doFlushInternal(flushDocStores, flushDeletes);
+      try {
+        return doFlushInternal(flushDocStores, flushDeletes);
+      } finally {
+        docWriter.balanceRAM();
+      }
     } finally {
       docWriter.clearFlushPending();
     }
diff --git a/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java b/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java
index ebd5108d2cb..092604e70bd 100644
--- a/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java
+++ b/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java
@@ -43,6 +43,8 @@ final class ReusableStringReader extends Reader {
       left -= len;
       return len;
     } else if (0 == left) {
+      // don't keep a reference (s could have been very large)
+      s = null;
       return -1;
     } else {
       s.getChars(upto, upto+left, c, off);
diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
index 5a804b7337d..3eb1f238219 100644
--- a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
+++ b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
@@ -190,7 +190,6 @@ final class TermVectorsTermsWriter extends TermsHashConsumer {
         lastPos = pos;
       }
       perDoc.perDocTvf.writeTo(tvf);
-      perDoc.perDocTvf.reset();
       perDoc.numVectorFields = 0;
     }
 
@@ -198,6 +197,7 @@ final class TermVectorsTermsWriter extends TermsHashConsumer {
 
     lastDocID++;
 
+    perDoc.reset();
     free(perDoc);
     assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument end");
   }