Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-2297: allow reader pooling to be enabled (through IndexWriterConfig) even if you're not pulling NRT readers 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/branches/newtrunk@926020 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2010-03-22 10:21:07 +00:00
parent 5f3e28a6f8
commit 8af45aadd2
5 changed files with 49 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/CHANGES.txt
View File

@ -193,6 +193,10 @@ New features
documents (previously this was hardwired to 5), using
IndexWriterConfig.setMaxThreadStates. (Mike McCandless)
* LUCENE-2297: Enable turning on reader pooling inside IndexWriter
even when getReader (near-real-timer reader) is not in use, through
IndexWriterConfig.enable/disableReaderPooling. (Mike McCandless)
Optimizations
* LUCENE-2075: Terms dict cache is now shared across threads instead

3
lucene/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -1081,7 +1081,8 @@ public class IndexWriter implements Closeable {
mergePolicy.setIndexWriter(this);
mergeScheduler = conf.getMergeScheduler();
mergedSegmentWarmer = conf.getMergedSegmentWarmer();
poolReaders = conf.getReaderPooling();
OpenMode mode = conf.getOpenMode();
boolean create;
if (mode == OpenMode.CREATE) {

39
lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
View File

@ -84,6 +84,9 @@ public final class IndexWriterConfig implements Cloneable {
* others to finish. */
public final static int DEFAULT_MAX_THREAD_STATES = 8;
/** Default setting for {@link #setIndexWriterPooling}. */
public final static boolean DEFAULT_READER_POOLING = false;
/**
* Sets the default (for any instance) maximum time to wait for a write lock
* (in milliseconds).
@ -118,6 +121,7 @@ public final class IndexWriterConfig implements Cloneable {
private IndexReaderWarmer mergedSegmentWarmer;
private MergePolicy mergePolicy;
private int maxThreadStates;
private boolean readerPooling;
// required for clone
private Version matchVersion;
@ -147,6 +151,7 @@ public final class IndexWriterConfig implements Cloneable {
mergedSegmentWarmer = null;
mergePolicy = new LogByteSizeMergePolicy();
maxThreadStates = DEFAULT_MAX_THREAD_STATES;
readerPooling = DEFAULT_READER_POOLING;
}
@Override
@ -505,6 +510,15 @@ public final class IndexWriterConfig implements Cloneable {
return this;
}
/**
* Returns the current MergePolicy in use by this writer.
*
* @see #setMergePolicy(MergePolicy)
*/
public MergePolicy getMergePolicy() {
return mergePolicy;
}
/**
* Sets the max number of simultaneous threads that may be indexing documents
* at once in IndexWriter. Values < 1 are invalid and if passed
@ -522,13 +536,23 @@ public final class IndexWriterConfig implements Cloneable {
return maxThreadStates;
}
/**
* Returns the current MergePolicy in use by this writer.
*
* @see #setMergePolicy(MergePolicy)
*/
public MergePolicy getMergePolicy() {
return mergePolicy;
/** By default, IndexWriter does not pool the
* SegmentReaders it must open for deletions and
* merging, unless a near-real-time reader has been
* obtained by calling {@link IndexWriter#getReader}.
* This method lets you enable pooling without getting a
* near-real-time reader. NOTE: if you set this to
* false, IndexWriter will still pool readers once
* {@link IndexWriter#getReader} is called. */
public IndexWriterConfig setReaderPooling(boolean readerPooling) {
this.readerPooling = readerPooling;
return this;
}
/** Returns true if IndexWriter should pool readers even
* if {@link IndexWriter#getReader} has not been called. */
public boolean getReaderPooling() {
return readerPooling;
}
/** Expert: sets the {@link DocConsumer} chain to be used to process documents. */
@ -562,6 +586,7 @@ public final class IndexWriterConfig implements Cloneable {
sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n");
sb.append("mergePolicy=").append(mergePolicy).append("\n");
sb.append("maxThreadStates=").append(maxThreadStates).append("\n");
sb.append("readerPooling=").append(readerPooling).append("\n");
return sb.toString();
}
}

3
lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
View File

@ -78,6 +78,7 @@ public class TestIndexWriterConfig extends LuceneTestCaseJ4 {
assertEquals(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS, conf.getMaxBufferedDeleteTerms());
assertEquals(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB, conf.getRAMBufferSizeMB(), 0.0);
assertEquals(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS, conf.getMaxBufferedDocs());
assertEquals(IndexWriterConfig.DEFAULT_READER_POOLING, conf.getReaderPooling());
assertTrue(DocumentsWriter.defaultIndexingChain == conf.getIndexingChain());
assertNull(conf.getMergedSegmentWarmer());
assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates());
@ -102,6 +103,7 @@ public class TestIndexWriterConfig extends LuceneTestCaseJ4 {
getters.add("getMergedSegmentWarmer");
getters.add("getMergePolicy");
getters.add("getMaxThreadStates");
getters.add("getReaderPooling");
for (Method m : IndexWriterConfig.class.getDeclaredMethods()) {
if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get")) {
assertTrue("method " + m.getName() + " is not tested for defaults", getters.contains(m.getName()));
@ -133,6 +135,7 @@ public class TestIndexWriterConfig extends LuceneTestCaseJ4 {
assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS);
assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS);
assertEquals(16.0, IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB, 0.0);
assertEquals(false, IndexWriterConfig.DEFAULT_READER_POOLING);
assertEquals(8, IndexWriterConfig.DEFAULT_MAX_THREAD_STATES);
}

12
lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
View File

@ -85,7 +85,8 @@ public class TestStressIndexing2 extends LuceneTestCase {
Directory dir2 = new MockRAMDirectory();
// mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1);
int maxThreadStates = 1+r.nextInt(10);
Map<String,Document> docs = indexRandom(10, 10, 100, dir1, maxThreadStates);
boolean doReaderPooling = r.nextBoolean();
Map<String,Document> docs = indexRandom(10, 10, 100, dir1, maxThreadStates, doReaderPooling);
indexSerial(docs, dir2);
// verifying verify
@ -103,6 +104,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
mergeFactor=r.nextInt(3)+2;
maxBufferedDocs=r.nextInt(3)+2;
int maxThreadStates = 1+r.nextInt(10);
boolean doReaderPooling = r.nextBoolean();
seed++;
int nThreads=r.nextInt(5)+1;
@ -110,7 +112,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
int range=r.nextInt(20)+1;
Directory dir1 = new MockRAMDirectory();
Directory dir2 = new MockRAMDirectory();
Map<String,Document> docs = indexRandom(nThreads, iter, range, dir1, maxThreadStates);
Map<String,Document> docs = indexRandom(nThreads, iter, range, dir1, maxThreadStates, doReaderPooling);
indexSerial(docs, dir2);
verifyEquals(dir1, dir2, "id");
}
@ -184,12 +186,14 @@ public class TestStressIndexing2 extends LuceneTestCase {
return dw;
}
public Map<String,Document> indexRandom(int nThreads, int iterations, int range, Directory dir, int maxThreadStates) throws IOException, InterruptedException {
public Map<String,Document> indexRandom(int nThreads, int iterations, int range, Directory dir, int maxThreadStates,
boolean doReaderPooling) throws IOException, InterruptedException {
Map<String,Document> docs = new HashMap<String,Document>();
for(int iter=0;iter<3;iter++) {
IndexWriter w = new MockIndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE)
.setRAMBufferSizeMB(0.1).setMaxBufferedDocs(maxBufferedDocs).setMaxThreadStates(maxThreadStates));
.setRAMBufferSizeMB(0.1).setMaxBufferedDocs(maxBufferedDocs).setMaxThreadStates(maxThreadStates)
.setReaderPooling(doReaderPooling));
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setUseCompoundFile(false);
lmp.setUseCompoundDocStore(false);