Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3601: fix addIndexes(3.x index) where we get a broken CFS, termvectors unreferenced files on init failures, and crash all indexes before checkindex in tests 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1207291 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2011-11-28 15:21:02 +00:00
parent 739ff5dacc
commit a629352512
7 changed files with 49 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
lucene/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -2344,10 +2344,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
* <p>
* <b>NOTE:</b> this method only copies the segments of the incoming indexes
* and does not merge them. Therefore deleted documents are not removed and
* the new segments are not merged with the existing ones. Also, if the merge
* policy allows compound files, then any segment that is not compound is
* converted to such. However, if the segment is compound, it is copied as-is
* even if the merge policy does not allow compound files.
* the new segments are not merged with the existing ones.
*
* <p>This requires this index not be among those to be added.
*
@ -2369,9 +2366,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
infoStream.message("IW", "flush at addIndexes(Directory...)");
flush(false, true);
int docCount = 0;
List<SegmentInfo> infos = new ArrayList<SegmentInfo>();
Comparator<String> versionComparator = StringHelper.getVersionComparator();
for (Directory dir : dirs) {
if (infoStream != null) {
infoStream.message("IW", "addIndexes: process directory " + dir);
@ -2383,7 +2378,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
for (SegmentInfo info : sis) {
assert !infos.contains(info): "dup info dir=" + info.dir + " name=" + info.name;
docCount += info.docCount;
String newSegName = newSegmentName();
String dsName = info.getDocStoreSegment();
@ -2391,23 +2385,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
infoStream.message("IW", "addIndexes: process segment origName=" + info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
}
// create CFS only if the source segment is not CFS, and MP agrees it
// should be CFS.
boolean createCFS;
synchronized (this) { // Guard segmentInfos
createCFS = !info.getUseCompoundFile()
&& mergePolicy.useCompoundFile(segmentInfos, info)
// optimize case only for segments that don't share doc stores
&& versionComparator.compare(info.getVersion(), "3.1") >= 0;
}
IOContext context = new IOContext(new MergeInfo(info.docCount, info.sizeInBytes(), true, -1));
if (createCFS) {
copySegmentIntoCFS(info, newSegName, context);
} else {
copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied, context);
}
copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied, context);
infos.add(info);
}
@ -2515,31 +2495,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
}
}
/** Copies the segment into the IndexWriter's directory, as a compound segment. */
private void copySegmentIntoCFS(SegmentInfo info, String segName, IOContext context) throws IOException {
String segFileName = IndexFileNames.segmentFileName(segName, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
Collection<String> files = info.files();
final CompoundFileDirectory cfsdir = new CompoundFileDirectory(directory, segFileName, context, true);
try {
for (String file : files) {
String newFileName = segName + IndexFileNames.stripSegmentName(file);
if (!IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
&& !IndexFileNames.isSeparateNormsFile(file)) {
info.dir.copy(cfsdir, file, file, context);
} else {
assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists";
info.dir.copy(directory, file, newFileName, context);
}
}
} finally {
IOUtils.close(cfsdir);
}
info.dir = directory;
info.name = segName;
info.setUseCompoundFile(true);
}
/** Copies the segment files as-is into the IndexWriter's directory. */
private void copySegmentAsIs(SegmentInfo info, String segName,
Map<String, String> dsNames, Set<String> dsFilesCopied, IOContext context)

2
lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java
View File

@ -64,7 +64,7 @@ public final class DefaultTermVectorsWriter extends TermVectorsWriter {
success = true;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(tvx, tvd, tvf);
abort();
}
}
}

10
lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsWriter.java
View File

@ -63,7 +63,15 @@ public class SimpleTextTermVectorsWriter extends TermVectorsWriter {
public SimpleTextTermVectorsWriter(Directory directory, String segment, IOContext context) throws IOException {
this.directory = directory;
this.segment = segment;
out = directory.createOutput(IndexFileNames.segmentFileName(segment, "", VECTORS_EXTENSION), context);
boolean success = false;
try {
out = directory.createOutput(IndexFileNames.segmentFileName(segment, "", VECTORS_EXTENSION), context);
success = true;
} finally {
if (!success) {
abort();
}
}
}
@Override

25
lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
View File

@ -21,6 +21,7 @@ import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@ -196,10 +197,13 @@ public class MockDirectoryWrapper extends Directory {
while(it.hasNext()) {
String name = it.next();
int damage = randomState.nextInt(4);
int damage = randomState.nextInt(5);
String action = null;
if (damage == 0) {
action = "deleted";
deleteFile(name, true);
} else if (damage == 1) {
action = "zeroed";
// Zero out file entirely
long length = fileLength(name);
byte[] zeroes = new byte[256];
@ -212,17 +216,25 @@ public class MockDirectoryWrapper extends Directory {
}
out.close();
} else if (damage == 2) {
action = "partially truncated";
// Partially Truncate the file:
IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState));
out.setLength(fileLength(name)/2);
out.close();
} else if (damage == 3) {
// The file survived intact:
action = "didn't change";
} else {
action = "fully truncated";
// Totally truncate the file to zero bytes
deleteFile(name, true);
IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState));
out.setLength(0);
out.close();
}
if (LuceneTestCase.VERBOSE) {
System.out.println("MockDirectoryWrapper: " + action + " unsynced file: " + name);
}
}
}
@ -504,10 +516,15 @@ public class MockDirectoryWrapper extends Directory {
}
open = false;
if (checkIndexOnClose) {
if (LuceneTestCase.VERBOSE) {
System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
}
if (IndexReader.indexExists(this)) {
if (LuceneTestCase.VERBOSE) {
System.out.println("\nNOTE: MockDirectoryWrapper: now crash");
}
unSyncedFiles.remove("segments.gen"); // otherwise we add minutes to the tests: LUCENE-3605
crash(); // corrumpt any unsynced-files
if (LuceneTestCase.VERBOSE) {
System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
}
_TestUtil.checkIndex(this);
}
}

59
lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
View File

@ -1086,65 +1086,6 @@ public class TestAddIndexes extends LuceneTestCase {
dir.close();
}
// LUCENE-3126: tests that if a non-CFS segment is copied, it is converted to
// a CFS, given MP preferences
public void testCopyIntoCFS() throws Exception {
// create an index, no CFS (so we can assert that existing segments are not affected)
Directory target = newDirectory();
LogMergePolicy lmp = newLogMergePolicy(false);
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null).setMergePolicy(lmp);
IndexWriter w = new IndexWriter(target, conf);
w.addDocument(new Document());
w.commit();
assertFalse(w.segmentInfos.info(0).getUseCompoundFile());
// prepare second index, no-CFS too + .del file + separate norms file
Directory src = newDirectory();
LogMergePolicy lmp2 = newLogMergePolicy(false);
IndexWriterConfig conf2 = newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random)).setMergePolicy(lmp2);
IndexWriter w2 = new IndexWriter(src, conf2);
Document doc = new Document();
doc.add(new Field("c", "some text", TextField.TYPE_STORED));
w2.addDocument(doc);
doc = new Document();
doc.add(new StringField("d", "delete"));
w2.addDocument(doc);
w2.commit();
w2.deleteDocuments(new Term("d", "delete"));
w2.commit();
w2.close();
// create separate norms file
IndexReader r = IndexReader.open(src, false);
r.setNorm(0, "c", (byte) 1);
r.close();
assertTrue(".del file not found", src.fileExists("_0_1.del"));
assertTrue("separate norms file not found", src.fileExists("_0_1.s0"));
// Case 1: force 'CFS' on target
lmp.setUseCompoundFile(true);
lmp.setNoCFSRatio(1.0);
w.addIndexes(src);
w.commit();
assertFalse("existing segments should not be modified by addIndexes", w.segmentInfos.info(0).getUseCompoundFile());
assertTrue("segment should have been converted to a CFS by addIndexes", w.segmentInfos.info(1).getUseCompoundFile());
assertTrue(".del file not found", target.fileExists("_1_1.del"));
assertTrue("separate norms file not found", target.fileExists("_1_1.s0"));
// Case 2: LMP disallows CFS
lmp.setUseCompoundFile(false);
w.addIndexes(src);
w.commit();
assertFalse("segment should not have been converted to a CFS by addIndexes if MP disallows", w.segmentInfos.info(2).getUseCompoundFile());
w.close();
// cleanup
src.close();
target.close();
}
private static class UnRegisteredCodec extends Codec {
public UnRegisteredCodec() {
super("NotRegistered");

6
lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java
View File

@ -39,7 +39,7 @@ public class TestDirectoryReader extends LuceneTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
dir = createDirectory();
doc1 = new Document();
doc2 = new Document();
DocHelper.setupDoc(doc1);
@ -57,6 +57,10 @@ public class TestDirectoryReader extends LuceneTestCase {
dir.close();
super.tearDown();
}
protected Directory createDirectory() throws IOException {
return newDirectory();
}
protected IndexReader openReader() throws IOException {
IndexReader reader;

12
lucene/src/test/org/apache/lucene/index/TestMultiReader.java
View File

@ -19,10 +19,20 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
public class TestMultiReader extends TestDirectoryReader {
// TODO: files are never fsynced if you do what this test is doing,
// so the checkindex is disabled.
@Override
protected Directory createDirectory() throws IOException {
MockDirectoryWrapper mdw = newDirectory();
mdw.setCheckIndexOnClose(false);
return mdw;
}
@Override
protected IndexReader openReader() throws IOException {
IndexReader reader;