From d8fb251c054e30b3e4933e82e82b14af66f430f5 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Fri, 13 Jun 2014 11:41:19 +0000 Subject: [PATCH] LUCENE-5756: Implement Accountable from IndexWriter git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1602388 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 3 +++ .../lucene/index/BufferedUpdatesStream.java | 6 ++++-- .../apache/lucene/index/DocumentsWriter.java | 10 ++++++++-- .../index/DocumentsWriterDeleteQueue.java | 8 +++++--- .../index/DocumentsWriterFlushControl.java | 12 +++++++++--- .../org/apache/lucene/index/IndexWriter.java | 19 +++++++++---------- .../index/TestFlushByRamOrCountsPolicy.java | 2 +- 7 files changed, 39 insertions(+), 21 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index a8772e16361..e06e467f431 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -156,6 +156,9 @@ Changes in Backwards Compatibility Policy API Changes +* LUCENE-5756: IndexWriter now implements Accountable and IW#ramSizeInBytes() + has been deprecated infavor of IW#ramBytesUsed() (Simon Willnauer) + * LUCENE-5725: MoreLikeThis#like now accepts multiple values per field. The pre-existing method has been deprecated in favor of a variable arguments for the like text. (Alex Ksikes via Simon Willnauer) diff --git a/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java b/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java index b75bc4df6f5..1e649fa9431 100644 --- a/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java +++ b/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryWrapperFilter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.InfoStream; @@ -50,7 +51,7 @@ import org.apache.lucene.util.InfoStream; * track which BufferedDeletes packets to apply to any given * segment. */ -class BufferedUpdatesStream { +class BufferedUpdatesStream implements Accountable { // TODO: maybe linked list? private final List updates = new ArrayList<>(); @@ -111,7 +112,8 @@ class BufferedUpdatesStream { return numTerms.get(); } - public long bytesUsed() { + @Override + public long ramBytesUsed() { return bytesUsed.get(); } diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java index 62af0773775..6a3761487f6 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -34,6 +34,7 @@ import org.apache.lucene.index.IndexWriter.Event; import org.apache.lucene.search.Query; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Accountable; import org.apache.lucene.util.InfoStream; /** @@ -94,7 +95,7 @@ import org.apache.lucene.util.InfoStream; * or none") added to the index. */ -final class DocumentsWriter implements Closeable { +final class DocumentsWriter implements Closeable, Accountable { private final Directory directory; private volatile boolean closed; @@ -662,7 +663,12 @@ final class DocumentsWriter implements Closeable { private void putEvent(Event event) { events.add(event); } - + + @Override + public long ramBytesUsed() { + return flushControl.ramBytesUsed(); + } + static final class ApplyDeletesEvent implements Event { static final Event INSTANCE = new ApplyDeletesEvent(); private int instCount = 0; diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java index 95ebd780479..d1c05a2385b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java @@ -24,6 +24,7 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate; import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate; import org.apache.lucene.search.Query; +import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; /** @@ -66,7 +67,7 @@ import org.apache.lucene.util.BytesRef; * will also not be added to its private deletes neither to the global deletes. * */ -final class DocumentsWriterDeleteQueue { +final class DocumentsWriterDeleteQueue implements Accountable { private volatile Node tail; @@ -448,8 +449,9 @@ final class DocumentsWriterDeleteQueue { globalBufferLock.unlock(); } } - - public long bytesUsed() { + + @Override + public long ramBytesUsed() { return globalBufferedUpdates.bytesUsed.get(); } diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java index 8e05d0393e5..2db5078be08 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java @@ -19,14 +19,15 @@ package org.apache.lucene.index; import java.util.ArrayList; import java.util.IdentityHashMap; -import java.util.List; import java.util.Iterator; import java.util.LinkedList; +import java.util.List; import java.util.Queue; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; +import org.apache.lucene.util.Accountable; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.ThreadInterruptedException; @@ -42,7 +43,7 @@ import org.apache.lucene.util.ThreadInterruptedException; * {@link IndexWriterConfig#getRAMPerThreadHardLimitMB()} to prevent address * space exhaustion. */ -final class DocumentsWriterFlushControl { +final class DocumentsWriterFlushControl implements Accountable { private final long hardMaxBytesPerDWPT; private long activeBytes = 0; @@ -428,7 +429,12 @@ final class DocumentsWriterFlushControl { } public long getDeleteBytesUsed() { - return documentsWriter.deleteQueue.bytesUsed() + bufferedUpdatesStream.bytesUsed(); + return documentsWriter.deleteQueue.ramBytesUsed() + bufferedUpdatesStream.ramBytesUsed(); + } + + @Override + public long ramBytesUsed() { + return getDeleteBytesUsed() + netBytes(); } synchronized int numFlushingDWPT() { diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index 6902a915deb..f428b3a4fba 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -56,6 +56,7 @@ import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.MergeInfo; import org.apache.lucene.store.TrackingDirectoryWrapper; +import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; @@ -192,7 +193,7 @@ import org.apache.lucene.util.Version; * referenced by the "front" of the index). For this, IndexFileDeleter * keeps track of the last non commit checkpoint. */ -public class IndexWriter implements Closeable, TwoPhaseCommit{ +public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { private static final int UNBOUNDED_MAX_MERGE_SEGMENTS = -1; @@ -425,6 +426,12 @@ public class IndexWriter implements Closeable, TwoPhaseCommit{ return r; } + @Override + public final long ramBytesUsed() { + ensureOpen(); + return docWriter.ramBytesUsed(); + } + /** Holds shared SegmentReader instances. IndexWriter uses * SegmentReaders for 1) applying deletes, 2) doing * merges, 3) handing out a real-time reader. This pool @@ -3090,7 +3097,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit{ } applyAllDeletesAndUpdates(); } else if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "don't apply deletes now delTermCount=" + bufferedUpdatesStream.numTerms() + " bytesUsed=" + bufferedUpdatesStream.bytesUsed()); + infoStream.message("IW", "don't apply deletes now delTermCount=" + bufferedUpdatesStream.numTerms() + " bytesUsed=" + bufferedUpdatesStream.ramBytesUsed()); } } @@ -3120,14 +3127,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit{ bufferedUpdatesStream.prune(segmentInfos); } - /** Expert: Return the total size of all index files currently cached in memory. - * Useful for size management with flushRamDocs() - */ - public final long ramSizeInBytes() { - ensureOpen(); - return docWriter.flushControl.netBytes() + bufferedUpdatesStream.bytesUsed(); - } - // for testing only DocumentsWriter getDocsWriter() { boolean test = false; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java index 17fdd622f0a..e5207a9b95f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java @@ -316,7 +316,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase { while (pendingDocs.decrementAndGet() > -1) { Document doc = docs.nextDoc(); writer.addDocument(doc); - long newRamSize = writer.ramSizeInBytes(); + long newRamSize = writer.ramBytesUsed(); if (newRamSize != ramSize) { ramSize = newRamSize; }