improve tests; fix RAFDirectory.openInput to respect pending delete

This commit is contained in:
Mike McCandless 2016-02-05 12:19:24 -05:00
parent 85c546b742
commit 24f55abfd5
10 changed files with 29 additions and 15 deletions

View File

@ -754,9 +754,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* IO error * IO error
*/ */
public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException { public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
Directory unwrapped = FilterDirectory.unwrap(d); if (d instanceof FSDirectory && ((FSDirectory) d).checkPendingDeletions()) {
if (unwrapped instanceof FSDirectory && ((FSDirectory) unwrapped).checkPendingDeletions()) { throw new IllegalArgumentException("Directory " + d + " is still has pending deleted files; cannot initialize IndexWriter");
throw new IllegalArgumentException("Directory still has pending deleted files; cannot initialize IndexWriter");
} }
conf.setIndexWriter(this); // prevent reuse by other instances conf.setIndexWriter(this); // prevent reuse by other instances

View File

@ -280,11 +280,11 @@ public abstract class FSDirectory extends BaseDirectory {
@Override @Override
public void sync(Collection<String> names) throws IOException { public void sync(Collection<String> names) throws IOException {
ensureOpen(); ensureOpen();
maybeDeletePendingFiles();
for (String name : names) { for (String name : names) {
fsync(name); fsync(name);
} }
maybeDeletePendingFiles();
} }
@Override @Override
@ -293,11 +293,12 @@ public abstract class FSDirectory extends BaseDirectory {
if (pendingDeletes.contains(source)) { if (pendingDeletes.contains(source)) {
throw new NoSuchFileException("file \"" + source + "\" is pending delete and cannot be moved"); throw new NoSuchFileException("file \"" + source + "\" is pending delete and cannot be moved");
} }
maybeDeletePendingFiles(); pendingDeletes.remove(dest);
Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE); Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE);
// TODO: should we move directory fsync to a separate 'syncMetadata' method? // TODO: should we move directory fsync to a separate 'syncMetadata' method?
// for example, to improve listCommits(), IndexFileDeleter could also call that after deleting segments_Ns // for example, to improve listCommits(), IndexFileDeleter could also call that after deleting segments_Ns
IOUtils.fsync(directory, true); IOUtils.fsync(directory, true);
maybeDeletePendingFiles();
} }
@Override @Override
@ -327,6 +328,7 @@ public abstract class FSDirectory extends BaseDirectory {
throw new NoSuchFileException("file \"" + name + "\" is already pending delete"); throw new NoSuchFileException("file \"" + name + "\" is already pending delete");
} }
privateDeleteFile(name); privateDeleteFile(name);
maybeDeletePendingFiles();
} }
/** Tries to delete any pending deleted files, and returns true if /** Tries to delete any pending deleted files, and returns true if

View File

@ -2737,7 +2737,7 @@ public class TestIndexWriter extends LuceneTestCase {
try { try {
w = new IndexWriter(dir, iwc); w = new IndexWriter(dir, iwc);
} catch (IllegalArgumentException iae) { } catch (IllegalArgumentException iae) {
assertEquals("Directory still has pending deleted files; cannot initialize IndexWriter", iae.getMessage()); assertTrue(iae.getMessage().contains("still has pending deleted files; cannot initialize IndexWriter"));
} }
in.close(); in.close();
} }

View File

@ -26,10 +26,12 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.PrintStreamInfoStream; import org.apache.lucene.util.PrintStreamInfoStream;
import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.TestUtil;
@SuppressFileSystems("WindowsFS")
public class TestIndexWriterOutOfFileDescriptors extends LuceneTestCase { public class TestIndexWriterOutOfFileDescriptors extends LuceneTestCase {
public void test() throws Exception { public void test() throws Exception {
MockDirectoryWrapper dir = newMockFSDirectory(createTempDir("TestIndexWriterOutOfFileDescriptors")); MockDirectoryWrapper dir = newMockFSDirectory(createTempDir("TestIndexWriterOutOfFileDescriptors"));

View File

@ -48,7 +48,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
static int seed=0; static int seed=0;
public void testRandomIWReader() throws Throwable { public void testRandomIWReader() throws Throwable {
Directory dir = newDirectory(); Directory dir = newMaybeVirusCheckingDirectory();
// TODO: verify equals using IW.getReader // TODO: verify equals using IW.getReader
DocsAndWriter dw = indexRandomIWReader(5, 3, 100, dir); DocsAndWriter dw = indexRandomIWReader(5, 3, 100, dir);
@ -61,8 +61,8 @@ public class TestStressIndexing2 extends LuceneTestCase {
} }
public void testRandom() throws Throwable { public void testRandom() throws Throwable {
Directory dir1 = newDirectory(); Directory dir1 = newMaybeVirusCheckingDirectory();
Directory dir2 = newDirectory(); Directory dir2 = newMaybeVirusCheckingDirectory();
// mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1); // mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1);
boolean doReaderPooling = random().nextBoolean(); boolean doReaderPooling = random().nextBoolean();
Map<String,Document> docs = indexRandom(5, 3, 100, dir1, doReaderPooling); Map<String,Document> docs = indexRandom(5, 3, 100, dir1, doReaderPooling);

View File

@ -104,7 +104,7 @@ public class TestStressNRT extends LuceneTestCase {
List<Thread> threads = new ArrayList<>(); List<Thread> threads = new ArrayList<>();
Directory dir = newDirectory(); Directory dir = newMaybeVirusCheckingDirectory();
final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random()))); final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())));
writer.setDoRandomForceMergeAssert(false); writer.setDoRandomForceMergeAssert(false);

View File

@ -66,6 +66,7 @@ public class RAFDirectory extends FSDirectory {
@Override @Override
public IndexInput openInput(String name, IOContext context) throws IOException { public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen(); ensureOpen();
ensureCanRead(name);
final File path = directory.resolve(name).toFile(); final File path = directory.resolve(name).toFile();
RandomAccessFile raf = new RandomAccessFile(path, "r"); RandomAccessFile raf = new RandomAccessFile(path, "r");
return new RAFIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context); return new RAFIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context);

View File

@ -28,8 +28,9 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
/** /**
* Acts like Windows, where random programs may open the files you just wrote in an unfriendly * Acts like a virus checker on Windows, where random programs may open the files you just wrote in an unfriendly
* way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc. * way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc. This is more evil
* than WindowsFS which just prevents deletion of files you still old open.
*/ */
public class VirusCheckingFS extends FilterFileSystemProvider { public class VirusCheckingFS extends FilterFileSystemProvider {
@ -75,5 +76,5 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
super.delete(path); super.delete(path);
} }
// TODO: rename? createOutput? deleteIfExists? // TODO: we could be more evil here, e.g. rename, createOutput, deleteIfExists
} }

View File

@ -1282,8 +1282,15 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
// expected // expected
} }
// write the file again if (random().nextBoolean()) {
try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) { try (IndexOutput out = fsDir.createOutput(fileName + "z", IOContext.DEFAULT)) {
}
// Make sure we can rename onto the deleted file:
fsDir.renameFile(fileName + "z", fileName);
} else {
// write the file again
try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
}
} }
assertEquals(0, fsDir.fileLength(fileName)); assertEquals(0, fsDir.fileLength(fileName));
assertTrue(Arrays.asList(fsDir.listAll()).contains(fileName)); assertTrue(Arrays.asList(fsDir.listAll()).contains(fileName));

View File

@ -160,6 +160,8 @@ public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
// no unexpected exceptions are raised: // no unexpected exceptions are raised:
public void testStressLocks() throws Exception { public void testStressLocks() throws Exception {
Path tempPath = createTempDir(); Path tempPath = createTempDir();
assumeFalse("cannot handle buggy Files.delete", TestUtil.hasWindowsFS(tempPath));
Directory dir = getDirectory(tempPath); Directory dir = getDirectory(tempPath);
// First create a 1 doc index: // First create a 1 doc index: