Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-6117: make infostream usable again 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1646240 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-12-17 13:51:53 +00:00
parent 9ab0f64f07
commit 5258d624c3
7 changed files with 110 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
View File

@ -388,7 +388,7 @@ final class DocumentsWriter implements Closeable, Accountable {
writer.globalFieldNumberMap);
state.dwpt = new DocumentsWriterPerThread(writer.newSegmentName(),
directory, config, infoStream, deleteQueue, infos,
writer.pendingNumDocs);
writer.pendingNumDocs, writer.enableTestPoints);
}
}

7
lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
View File

@ -155,9 +155,10 @@ class DocumentsWriterPerThread {
final IntBlockPool.Allocator intBlockAllocator;
private final AtomicLong pendingNumDocs;
private final LiveIndexWriterConfig indexWriterConfig;
private final boolean enableTestPoints;
public DocumentsWriterPerThread(String segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue,
FieldInfos.Builder fieldInfos, AtomicLong pendingNumDocs) throws IOException {
FieldInfos.Builder fieldInfos, AtomicLong pendingNumDocs, boolean enableTestPoints) throws IOException {
this.directoryOrig = directory;
this.directory = new TrackingDirectoryWrapper(directory);
this.fieldInfos = fieldInfos;
@ -184,6 +185,7 @@ class DocumentsWriterPerThread {
// this should be the last call in the ctor
// it really sucks that we need to pull this within the ctor and pass this ref to the chain!
consumer = indexWriterConfig.getIndexingChain().getChain(this);
this.enableTestPoints = enableTestPoints;
}
public FieldInfos.Builder getFieldInfosBuilder() {
@ -191,7 +193,8 @@ class DocumentsWriterPerThread {
}
final void testPoint(String message) {
if (infoStream.isEnabled("TP")) {
if (enableTestPoints) {
assert infoStream.isEnabled("TP"); // don't enable unless you need them.
infoStream.message("TP", message);
}
}

6
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -214,6 +214,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
static int getActualMaxDocs() {
return IndexWriter.actualMaxDocs;
}
/** Used only for testing. */
boolean enableTestPoints = false;
private static final int UNBOUNDED_MAX_MERGE_SEGMENTS = -1;
@ -4417,7 +4420,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// startMergeInit
// DocumentsWriter.ThreadState.init start
private final void testPoint(String message) {
if (infoStream.isEnabled("TP")) {
if (enableTestPoints) {
assert infoStream.isEnabled("TP"); // don't enable unless you need them.
infoStream.message("TP", message);
}
}

28
lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
View File

@ -2428,7 +2428,7 @@ public class TestIndexWriter extends LuceneTestCase {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
final SetOnce<IndexWriter> iwRef = new SetOnce<>();
iwc.setInfoStream(new RandomIndexWriter.TestPointInfoStream(iwc.getInfoStream(), new RandomIndexWriter.TestPoint() {
IndexWriter evilWriter = RandomIndexWriter.mockIndexWriter(dir, iwc, new RandomIndexWriter.TestPoint() {
@Override
public void apply(String message) {
if ("startCommitMerge".equals(message)) {
@ -2437,8 +2437,7 @@ public class TestIndexWriter extends LuceneTestCase {
iwRef.get().setKeepFullyDeletedSegments(true);
}
}
}));
IndexWriter evilWriter = new IndexWriter(dir, iwc);
});
iwRef.set(evilWriter);
for (int i = 0; i < 1000; i++) {
addDoc(evilWriter);
@ -2646,10 +2645,12 @@ public class TestIndexWriter extends LuceneTestCase {
final CountDownLatch startCommit = new CountDownLatch(1);
final CountDownLatch finishCommit = new CountDownLatch(1);
// infostream that "takes a long time" to commit
InfoStream slowCommittingInfoStream = new InfoStream() {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(null);
// use an infostream that "takes a long time" to commit
final IndexWriter iw = RandomIndexWriter.mockIndexWriter(dir, iwc, new RandomIndexWriter.TestPoint() {
@Override
public void message(String component, String message) {
public void apply(String message) {
if (message.equals("finishStartCommit")) {
startCommit.countDown();
try {
@ -2659,20 +2660,7 @@ public class TestIndexWriter extends LuceneTestCase {
}
}
}
@Override
public boolean isEnabled(String component) {
return true;
}
@Override
public void close() throws IOException {}
};
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(null);
iwc.setInfoStream(slowCommittingInfoStream);
final IndexWriter iw = new IndexWriter(dir, iwc);
});
new Thread() {
@Override
public void run() {

84
lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java Normal file
View File

@ -0,0 +1,84 @@
package org.apache.lucene.index;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.document.Document;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
/** Tests indexwriter's infostream */
public class TestInfoStream extends LuceneTestCase {
/** we shouldn't have test points unless we ask */
public void testTestPointsOff() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(null);
iwc.setInfoStream(new InfoStream() {
@Override
public void close() throws IOException {}
@Override
public void message(String component, String message) {
assertFalse("TP".equals(component));
}
@Override
public boolean isEnabled(String component) {
assertFalse("TP".equals(component));
return true;
}
});
IndexWriter iw = new IndexWriter(dir, iwc);
iw.addDocument(new Document());
iw.close();
dir.close();
}
/** but they should work when we need */
public void testTestPointsOn() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(null);
AtomicBoolean seenTestPoint = new AtomicBoolean();
iwc.setInfoStream(new InfoStream() {
@Override
public void close() throws IOException {}
@Override
public void message(String component, String message) {
if ("TP".equals(component)) {
seenTestPoint.set(true);
}
}
@Override
public boolean isEnabled(String component) {
return true;
}
});
IndexWriter iw = new IndexWriter(dir, iwc);
iw.enableTestPoints = true;
iw.addDocument(new Document());
iw.close();
dir.close();
assertTrue(seenTestPoint.get());
}
}

14
lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
View File

@ -45,16 +45,6 @@ public class TestStressIndexing2 extends LuceneTestCase {
static int maxBufferedDocs=3;
static int seed=0;
public final class YieldTestPoint implements RandomIndexWriter.TestPoint {
@Override
public void apply(String name) {
// if (name.equals("startCommit")) {
if (random().nextInt(4) == 2)
Thread.yield();
}
}
//
public void testRandomIWReader() throws Throwable {
Directory dir = newDirectory();
@ -148,7 +138,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
.setOpenMode(OpenMode.CREATE)
.setRAMBufferSizeMB(0.1)
.setMaxBufferedDocs(maxBufferedDocs)
.setMergePolicy(newLogMergePolicy()), new YieldTestPoint());
.setMergePolicy(newLogMergePolicy()), random());
w.commit();
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setNoCFSRatio(0.0);
@ -203,7 +193,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
.setMaxBufferedDocs(maxBufferedDocs)
.setIndexerThreadPool(new DocumentsWriterPerThreadPool(maxThreadStates))
.setReaderPooling(doReaderPooling)
.setMergePolicy(newLogMergePolicy()), new YieldTestPoint());
.setMergePolicy(newLogMergePolicy()), random());
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setNoCFSRatio(0.0);
lmp.setMergeFactor(mergeFactor);

7
lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
View File

@ -50,7 +50,7 @@ public class RandomIndexWriter implements Closeable {
private boolean getReaderCalled;
private final Codec codec; // sugar
/** Returns an indexwriter that randomly mixes up thread scheduling (by yielding at test points) */
public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException {
// Randomly calls Thread.yield so we mixup thread scheduling
final Random random = new Random(r.nextLong());
@ -63,9 +63,12 @@ public class RandomIndexWriter implements Closeable {
});
}
/** Returns an indexwriter that enables the specified test point */
public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, TestPoint testPoint) throws IOException {
conf.setInfoStream(new TestPointInfoStream(conf.getInfoStream(), testPoint));
return new IndexWriter(dir, conf);
IndexWriter iw = new IndexWriter(dir, conf);
iw.enableTestPoints = true;
return iw;
}
/** create a RandomIndexWriter with a random config: Uses MockAnalyzer */