From 94530940e4de8b476a5886f284578c933a8f33ef Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 25 Jan 2017 16:15:04 +0100 Subject: [PATCH] LUCENE-7647: CompressingStoredFieldsFormat should reclaim memory more aggressively. --- lucene/CHANGES.txt | 4 ++ .../CompressingStoredFieldsWriter.java | 5 +- .../codecs/compressing/CompressionMode.java | 49 +++++++++++++------ .../lucene/codecs/compressing/Compressor.java | 3 +- .../dummy/DummyCompressingCodec.java | 3 ++ 5 files changed, 46 insertions(+), 18 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index a68d7e3d762..9a1622f27cd 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -125,6 +125,10 @@ Bug Fixes * LUCENE-7657: Fixed potential memory leak in the case that a (Span)TermQuery with a TermContext is cached. (Adrien Grand) +* LUCENE-7647: Made stored fields reclaim native memory more aggressively when + configured with BEST_COMPRESSION. This could otherwise result in out-of-memory + issues. (Adrien Grand) + ======================= Lucene 6.4.0 ======================= API Changes diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java index 7ab20afd9ef..5b42870fac7 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java @@ -81,7 +81,7 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter { private CompressingStoredFieldsIndexWriter indexWriter; private IndexOutput fieldsStream; - private final Compressor compressor; + private Compressor compressor; private final CompressionMode compressionMode; private final int chunkSize; private final int maxDocsPerChunk; @@ -141,10 +141,11 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter { @Override public void close() throws IOException { try { - IOUtils.close(fieldsStream, indexWriter); + IOUtils.close(fieldsStream, indexWriter, compressor); } finally { fieldsStream = null; indexWriter = null; + compressor = null; } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java index 326eba32086..53a84cbdd53 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java @@ -164,6 +164,10 @@ public abstract class CompressionMode { LZ4.compress(bytes, off, len, out, ht); } + @Override + public void close() throws IOException { + // no-op + } } private static final class LZ4HighCompressor extends Compressor { @@ -180,15 +184,17 @@ public abstract class CompressionMode { LZ4.compressHC(bytes, off, len, out, ht); } + @Override + public void close() throws IOException { + // no-op + } } private static final class DeflateDecompressor extends Decompressor { - final Inflater decompressor; byte[] compressed; DeflateDecompressor() { - decompressor = new Inflater(true); compressed = new byte[0]; } @@ -207,20 +213,24 @@ public abstract class CompressionMode { in.readBytes(compressed, 0, compressedLength); compressed[compressedLength] = 0; // explicitly set dummy byte to 0 - decompressor.reset(); - // extra "dummy byte" - decompressor.setInput(compressed, 0, paddedLength); - - bytes.offset = bytes.length = 0; - bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength); + final Inflater decompressor = new Inflater(true); try { - bytes.length = decompressor.inflate(bytes.bytes, bytes.length, originalLength); - } catch (DataFormatException e) { - throw new IOException(e); - } - if (!decompressor.finished()) { - throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput() - + ", needsDict=" + decompressor.needsDictionary(), in); + // extra "dummy byte" + decompressor.setInput(compressed, 0, paddedLength); + + bytes.offset = bytes.length = 0; + bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength); + try { + bytes.length = decompressor.inflate(bytes.bytes, bytes.length, originalLength); + } catch (DataFormatException e) { + throw new IOException(e); + } + if (!decompressor.finished()) { + throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput() + + ", needsDict=" + decompressor.needsDictionary(), in); + } + } finally { + decompressor.end(); } if (bytes.length != originalLength) { throw new CorruptIndexException("Lengths mismatch: " + bytes.length + " != " + originalLength, in); @@ -240,6 +250,7 @@ public abstract class CompressionMode { final Deflater compressor; byte[] compressed; + boolean closed; DeflateCompressor(int level) { compressor = new Deflater(level, true); @@ -275,6 +286,14 @@ public abstract class CompressionMode { out.writeBytes(compressed, totalCount); } + @Override + public void close() throws IOException { + if (closed == false) { + compressor.end(); + closed = true; + } + } + } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java index bd2fadbd720..f95246c217f 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java @@ -17,6 +17,7 @@ package org.apache.lucene.codecs.compressing; +import java.io.Closeable; import java.io.IOException; import org.apache.lucene.store.DataOutput; @@ -24,7 +25,7 @@ import org.apache.lucene.store.DataOutput; /** * A data compressor. */ -public abstract class Compressor { +public abstract class Compressor implements Closeable { /** Sole constructor, typically called from sub-classes. */ protected Compressor() {} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java index d15adadd4f7..167418e0f9d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java @@ -79,6 +79,9 @@ public class DummyCompressingCodec extends CompressingCodec { out.writeBytes(bytes, off, len); } + @Override + public void close() throws IOException {}; + }; /** Constructor that allows to configure the chunk size. */