From 0d37e4dc768fdb50ba030d69447c927687a7b70f Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 3 Sep 2020 11:37:34 +0200 Subject: [PATCH] LUCENE-9483: Disable per-thread caching of buffers for decompression of stored fields. (#1787) These buffers can use lots of memory when the number of segments, threads or both is high. --- .../CompressingStoredFieldsReader.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java index 91916ddb951..cd9197f1a34 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java @@ -405,8 +405,17 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader { // the start pointer at which you can read the compressed documents private long startPointer; - private final BytesRef spare = new BytesRef(); - private final BytesRef bytes = new BytesRef(); + private final BytesRef spare; + private final BytesRef bytes; + + BlockState() { + if (merging) { + spare = new BytesRef(); + bytes = new BytesRef(); + } else { + spare = bytes = null; + } + } boolean contains(int docID) { return docID >= docBase && docID < docBase + chunkDocs; @@ -536,6 +545,13 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader { final int totalLength = offsets[chunkDocs]; final int numStoredFields = this.numStoredFields[index]; + final BytesRef bytes; + if (merging) { + bytes = this.bytes; + } else { + bytes = new BytesRef(); + } + final DataInput documentInput; if (length == 0) { // empty