mirror of https://github.com/apache/lucene.git
improve tests; fix RAFDirectory.openInput to respect pending delete
This commit is contained in:
parent
85c546b742
commit
24f55abfd5
|
@ -754,9 +754,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
|
|||
* IO error
|
||||
*/
|
||||
public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
|
||||
Directory unwrapped = FilterDirectory.unwrap(d);
|
||||
if (unwrapped instanceof FSDirectory && ((FSDirectory) unwrapped).checkPendingDeletions()) {
|
||||
throw new IllegalArgumentException("Directory still has pending deleted files; cannot initialize IndexWriter");
|
||||
if (d instanceof FSDirectory && ((FSDirectory) d).checkPendingDeletions()) {
|
||||
throw new IllegalArgumentException("Directory " + d + " is still has pending deleted files; cannot initialize IndexWriter");
|
||||
}
|
||||
|
||||
conf.setIndexWriter(this); // prevent reuse by other instances
|
||||
|
|
|
@ -280,11 +280,11 @@ public abstract class FSDirectory extends BaseDirectory {
|
|||
@Override
|
||||
public void sync(Collection<String> names) throws IOException {
|
||||
ensureOpen();
|
||||
maybeDeletePendingFiles();
|
||||
|
||||
for (String name : names) {
|
||||
fsync(name);
|
||||
}
|
||||
maybeDeletePendingFiles();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -293,11 +293,12 @@ public abstract class FSDirectory extends BaseDirectory {
|
|||
if (pendingDeletes.contains(source)) {
|
||||
throw new NoSuchFileException("file \"" + source + "\" is pending delete and cannot be moved");
|
||||
}
|
||||
maybeDeletePendingFiles();
|
||||
pendingDeletes.remove(dest);
|
||||
Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE);
|
||||
// TODO: should we move directory fsync to a separate 'syncMetadata' method?
|
||||
// for example, to improve listCommits(), IndexFileDeleter could also call that after deleting segments_Ns
|
||||
IOUtils.fsync(directory, true);
|
||||
maybeDeletePendingFiles();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -327,6 +328,7 @@ public abstract class FSDirectory extends BaseDirectory {
|
|||
throw new NoSuchFileException("file \"" + name + "\" is already pending delete");
|
||||
}
|
||||
privateDeleteFile(name);
|
||||
maybeDeletePendingFiles();
|
||||
}
|
||||
|
||||
/** Tries to delete any pending deleted files, and returns true if
|
||||
|
|
|
@ -2737,7 +2737,7 @@ public class TestIndexWriter extends LuceneTestCase {
|
|||
try {
|
||||
w = new IndexWriter(dir, iwc);
|
||||
} catch (IllegalArgumentException iae) {
|
||||
assertEquals("Directory still has pending deleted files; cannot initialize IndexWriter", iae.getMessage());
|
||||
assertTrue(iae.getMessage().contains("still has pending deleted files; cannot initialize IndexWriter"));
|
||||
}
|
||||
in.close();
|
||||
}
|
||||
|
|
|
@ -26,10 +26,12 @@ import org.apache.lucene.store.Directory;
|
|||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.store.MockDirectoryWrapper;
|
||||
import org.apache.lucene.util.LineFileDocs;
|
||||
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.lucene.util.PrintStreamInfoStream;
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
|
||||
@SuppressFileSystems("WindowsFS")
|
||||
public class TestIndexWriterOutOfFileDescriptors extends LuceneTestCase {
|
||||
public void test() throws Exception {
|
||||
MockDirectoryWrapper dir = newMockFSDirectory(createTempDir("TestIndexWriterOutOfFileDescriptors"));
|
||||
|
|
|
@ -48,7 +48,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
|
|||
static int seed=0;
|
||||
|
||||
public void testRandomIWReader() throws Throwable {
|
||||
Directory dir = newDirectory();
|
||||
Directory dir = newMaybeVirusCheckingDirectory();
|
||||
|
||||
// TODO: verify equals using IW.getReader
|
||||
DocsAndWriter dw = indexRandomIWReader(5, 3, 100, dir);
|
||||
|
@ -61,8 +61,8 @@ public class TestStressIndexing2 extends LuceneTestCase {
|
|||
}
|
||||
|
||||
public void testRandom() throws Throwable {
|
||||
Directory dir1 = newDirectory();
|
||||
Directory dir2 = newDirectory();
|
||||
Directory dir1 = newMaybeVirusCheckingDirectory();
|
||||
Directory dir2 = newMaybeVirusCheckingDirectory();
|
||||
// mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1);
|
||||
boolean doReaderPooling = random().nextBoolean();
|
||||
Map<String,Document> docs = indexRandom(5, 3, 100, dir1, doReaderPooling);
|
||||
|
|
|
@ -104,7 +104,7 @@ public class TestStressNRT extends LuceneTestCase {
|
|||
|
||||
List<Thread> threads = new ArrayList<>();
|
||||
|
||||
Directory dir = newDirectory();
|
||||
Directory dir = newMaybeVirusCheckingDirectory();
|
||||
|
||||
final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())));
|
||||
writer.setDoRandomForceMergeAssert(false);
|
||||
|
|
|
@ -66,6 +66,7 @@ public class RAFDirectory extends FSDirectory {
|
|||
@Override
|
||||
public IndexInput openInput(String name, IOContext context) throws IOException {
|
||||
ensureOpen();
|
||||
ensureCanRead(name);
|
||||
final File path = directory.resolve(name).toFile();
|
||||
RandomAccessFile raf = new RandomAccessFile(path, "r");
|
||||
return new RAFIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context);
|
||||
|
|
|
@ -28,8 +28,9 @@ import org.apache.lucene.index.IndexWriter;
|
|||
import org.apache.lucene.util.LuceneTestCase;
|
||||
|
||||
/**
|
||||
* Acts like Windows, where random programs may open the files you just wrote in an unfriendly
|
||||
* way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc.
|
||||
* Acts like a virus checker on Windows, where random programs may open the files you just wrote in an unfriendly
|
||||
* way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc. This is more evil
|
||||
* than WindowsFS which just prevents deletion of files you still old open.
|
||||
*/
|
||||
public class VirusCheckingFS extends FilterFileSystemProvider {
|
||||
|
||||
|
@ -75,5 +76,5 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
|
|||
super.delete(path);
|
||||
}
|
||||
|
||||
// TODO: rename? createOutput? deleteIfExists?
|
||||
// TODO: we could be more evil here, e.g. rename, createOutput, deleteIfExists
|
||||
}
|
||||
|
|
|
@ -1282,8 +1282,15 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
|
|||
// expected
|
||||
}
|
||||
|
||||
// write the file again
|
||||
try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
|
||||
if (random().nextBoolean()) {
|
||||
try (IndexOutput out = fsDir.createOutput(fileName + "z", IOContext.DEFAULT)) {
|
||||
}
|
||||
// Make sure we can rename onto the deleted file:
|
||||
fsDir.renameFile(fileName + "z", fileName);
|
||||
} else {
|
||||
// write the file again
|
||||
try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
|
||||
}
|
||||
}
|
||||
assertEquals(0, fsDir.fileLength(fileName));
|
||||
assertTrue(Arrays.asList(fsDir.listAll()).contains(fileName));
|
||||
|
|
|
@ -160,6 +160,8 @@ public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
|
|||
// no unexpected exceptions are raised:
|
||||
public void testStressLocks() throws Exception {
|
||||
Path tempPath = createTempDir();
|
||||
assumeFalse("cannot handle buggy Files.delete", TestUtil.hasWindowsFS(tempPath));
|
||||
|
||||
Directory dir = getDirectory(tempPath);
|
||||
|
||||
// First create a 1 doc index:
|
||||
|
|
Loading…
Reference in New Issue