LUCENE-7647: CompressingStoredFieldsFormat should reclaim memory more aggressively.

lucene/CHANGES.txt

@@ -125,6 +125,10 @@ Bug Fixes
 * LUCENE-7657: Fixed potential memory leak in the case that a (Span)TermQuery
   with a TermContext is cached. (Adrien Grand)
 
+* LUCENE-7647: Made stored fields reclaim native memory more aggressively when
+  configured with BEST_COMPRESSION. This could otherwise result in out-of-memory
+  issues. (Adrien Grand)
+
 ======================= Lucene 6.4.0 =======================
 
 API Changes

lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java

@@ -81,7 +81,7 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
   private CompressingStoredFieldsIndexWriter indexWriter;
   private IndexOutput fieldsStream;
 
-  private final Compressor compressor;
+  private Compressor compressor;
   private final CompressionMode compressionMode;
   private final int chunkSize;
   private final int maxDocsPerChunk;
@@ -141,10 +141,11 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
   @Override
   public void close() throws IOException {
     try {
-      IOUtils.close(fieldsStream, indexWriter);
+      IOUtils.close(fieldsStream, indexWriter, compressor);
     } finally {
       fieldsStream = null;
       indexWriter = null;
+      compressor = null;
     }
   }
 

lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java

@@ -164,6 +164,10 @@ public abstract class CompressionMode {
       LZ4.compress(bytes, off, len, out, ht);
     }
 
+    @Override
+    public void close() throws IOException {
+      // no-op
+    }
   }
 
   private static final class LZ4HighCompressor extends Compressor {
@@ -180,15 +184,17 @@ public abstract class CompressionMode {
       LZ4.compressHC(bytes, off, len, out, ht);
     }
 
+    @Override
+    public void close() throws IOException {
+      // no-op
+    }
   }
 
   private static final class DeflateDecompressor extends Decompressor {
 
-    final Inflater decompressor;
     byte[] compressed;
 
     DeflateDecompressor() {
-      decompressor = new Inflater(true);
       compressed = new byte[0];
     }
 
@@ -207,20 +213,24 @@ public abstract class CompressionMode {
       in.readBytes(compressed, 0, compressedLength);
       compressed[compressedLength] = 0; // explicitly set dummy byte to 0
 
-      decompressor.reset();
-      // extra "dummy byte"
-      decompressor.setInput(compressed, 0, paddedLength);
-
-      bytes.offset = bytes.length = 0;
-      bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength);
+      final Inflater decompressor = new Inflater(true);
       try {
-        bytes.length = decompressor.inflate(bytes.bytes, bytes.length, originalLength);
-      } catch (DataFormatException e) {
-        throw new IOException(e);
-      }
-      if (!decompressor.finished()) {
-        throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput() 
-                                                            + ", needsDict=" + decompressor.needsDictionary(), in);
+        // extra "dummy byte"
+        decompressor.setInput(compressed, 0, paddedLength);
+
+        bytes.offset = bytes.length = 0;
+        bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength);
+        try {
+          bytes.length = decompressor.inflate(bytes.bytes, bytes.length, originalLength);
+        } catch (DataFormatException e) {
+          throw new IOException(e);
+        }
+        if (!decompressor.finished()) {
+          throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput()
+                                                              + ", needsDict=" + decompressor.needsDictionary(), in);
+        }
+      } finally {
+        decompressor.end();
       }
       if (bytes.length != originalLength) {
         throw new CorruptIndexException("Lengths mismatch: " + bytes.length + " != " + originalLength, in);
@@ -240,6 +250,7 @@ public abstract class CompressionMode {
 
     final Deflater compressor;
     byte[] compressed;
+    boolean closed;
 
     DeflateCompressor(int level) {
       compressor = new Deflater(level, true);
@@ -275,6 +286,14 @@ public abstract class CompressionMode {
       out.writeBytes(compressed, totalCount);
     }
 
+    @Override
+    public void close() throws IOException {
+      if (closed == false) {
+        compressor.end();
+        closed = true;
+      }
+    }
+
   }
 
 }

lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java

@@ -17,6 +17,7 @@
 package org.apache.lucene.codecs.compressing;
 
 
+import java.io.Closeable;
 import java.io.IOException;
 
 import org.apache.lucene.store.DataOutput;
@@ -24,7 +25,7 @@ import org.apache.lucene.store.DataOutput;
 /**
  * A data compressor.
  */
-public abstract class Compressor {
+public abstract class Compressor implements Closeable {
 
   /** Sole constructor, typically called from sub-classes. */
   protected Compressor() {}

lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java

@@ -79,6 +79,9 @@ public class DummyCompressingCodec extends CompressingCodec {
       out.writeBytes(bytes, off, len);
     }
 
+    @Override
+    public void close() throws IOException {};
+
   };
 
   /** Constructor that allows to configure the chunk size. */