improve tests; fix RAFDirectory.openInput to respect pending delete

lucene/core/src/java/org/apache/lucene/index/IndexWriter.java

@@ -754,9 +754,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
    *           IO error
    */
   public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
-    Directory unwrapped = FilterDirectory.unwrap(d);
+    if (d instanceof FSDirectory && ((FSDirectory) d).checkPendingDeletions()) {
-    if (unwrapped instanceof FSDirectory && ((FSDirectory) unwrapped).checkPendingDeletions()) {
+      throw new IllegalArgumentException("Directory " + d + " is still has pending deleted files; cannot initialize IndexWriter");
-      throw new IllegalArgumentException("Directory still has pending deleted files; cannot initialize IndexWriter");
     }
 
     conf.setIndexWriter(this); // prevent reuse by other instances

lucene/core/src/java/org/apache/lucene/store/FSDirectory.java

@@ -280,11 +280,11 @@ public abstract class FSDirectory extends BaseDirectory {
   @Override
   public void sync(Collection<String> names) throws IOException {
     ensureOpen();
-    maybeDeletePendingFiles();
 
     for (String name : names) {
       fsync(name);
     }
+    maybeDeletePendingFiles();
   }
 
   @Override
@@ -293,11 +293,12 @@ public abstract class FSDirectory extends BaseDirectory {
     if (pendingDeletes.contains(source)) {
       throw new NoSuchFileException("file \"" + source + "\" is pending delete and cannot be moved");
     }
-    maybeDeletePendingFiles();
+    pendingDeletes.remove(dest);
     Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE);
     // TODO: should we move directory fsync to a separate 'syncMetadata' method?
     // for example, to improve listCommits(), IndexFileDeleter could also call that after deleting segments_Ns
     IOUtils.fsync(directory, true);
+    maybeDeletePendingFiles();
   }
 
   @Override
@@ -327,6 +328,7 @@ public abstract class FSDirectory extends BaseDirectory {
       throw new NoSuchFileException("file \"" + name + "\" is already pending delete");
     }
     privateDeleteFile(name);
+    maybeDeletePendingFiles();
   }
 
   /** Tries to delete any pending deleted files, and returns true if

lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java

@@ -2737,7 +2737,7 @@ public class TestIndexWriter extends LuceneTestCase {
       try {
         w = new IndexWriter(dir, iwc);
       } catch (IllegalArgumentException iae) {
-        assertEquals("Directory still has pending deleted files; cannot initialize IndexWriter", iae.getMessage());
+        assertTrue(iae.getMessage().contains("still has pending deleted files; cannot initialize IndexWriter"));
       }
       in.close();
     }

lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java

@@ -26,10 +26,12 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LineFileDocs;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.PrintStreamInfoStream;
 import org.apache.lucene.util.TestUtil;
 
+@SuppressFileSystems("WindowsFS")
 public class TestIndexWriterOutOfFileDescriptors extends LuceneTestCase {
   public void test() throws Exception {
     MockDirectoryWrapper dir = newMockFSDirectory(createTempDir("TestIndexWriterOutOfFileDescriptors"));

lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java

@@ -48,7 +48,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
   static int seed=0;
 
   public void testRandomIWReader() throws Throwable {
-    Directory dir = newDirectory();
+    Directory dir = newMaybeVirusCheckingDirectory();
     
     // TODO: verify equals using IW.getReader
     DocsAndWriter dw = indexRandomIWReader(5, 3, 100, dir);
@@ -61,8 +61,8 @@ public class TestStressIndexing2 extends LuceneTestCase {
   }
   
   public void testRandom() throws Throwable {
-    Directory dir1 = newDirectory();
+    Directory dir1 = newMaybeVirusCheckingDirectory();
-    Directory dir2 = newDirectory();
+    Directory dir2 = newMaybeVirusCheckingDirectory();
     // mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1);
     boolean doReaderPooling = random().nextBoolean();
     Map<String,Document> docs = indexRandom(5, 3, 100, dir1, doReaderPooling);

lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java

@@ -104,7 +104,7 @@ public class TestStressNRT extends LuceneTestCase {
 
     List<Thread> threads = new ArrayList<>();
 
-    Directory dir = newDirectory();
+    Directory dir = newMaybeVirusCheckingDirectory();
 
     final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())));
     writer.setDoRandomForceMergeAssert(false);

lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java

@@ -66,6 +66,7 @@ public class RAFDirectory extends FSDirectory {
   @Override
   public IndexInput openInput(String name, IOContext context) throws IOException {
     ensureOpen();
+    ensureCanRead(name);
     final File path = directory.resolve(name).toFile();
     RandomAccessFile raf = new RandomAccessFile(path, "r");
     return new RAFIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context);

lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java

@@ -28,8 +28,9 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.util.LuceneTestCase;
 
 /** 
- * Acts like Windows, where random programs may open the files you just wrote in an unfriendly
+ * Acts like a virus checker on Windows, where random programs may open the files you just wrote in an unfriendly
- * way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc.
+ * way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc.  This is more evil
+ * than WindowsFS which just prevents deletion of files you still old open.
  */
 public class VirusCheckingFS extends FilterFileSystemProvider {
 
@@ -75,5 +76,5 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
     super.delete(path);
   }
 
-  // TODO: rename?  createOutput?  deleteIfExists?
+  // TODO: we could be more evil here, e.g. rename, createOutput, deleteIfExists
 }

lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java

@@ -1282,8 +1282,15 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
         // expected
       }
 
-      // write the file again
+      if (random().nextBoolean()) {
-      try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+        try (IndexOutput out = fsDir.createOutput(fileName + "z", IOContext.DEFAULT)) {
+        }
+        // Make sure we can rename onto the deleted file:
+        fsDir.renameFile(fileName + "z", fileName);
+      } else {
+        // write the file again
+        try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+        }
       }
       assertEquals(0, fsDir.fileLength(fileName));
       assertTrue(Arrays.asList(fsDir.listAll()).contains(fileName));

lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java

@@ -160,6 +160,8 @@ public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
   // no unexpected exceptions are raised:
   public void testStressLocks() throws Exception {
     Path tempPath = createTempDir();
+    assumeFalse("cannot handle buggy Files.delete", TestUtil.hasWindowsFS(tempPath));
+
     Directory dir = getDirectory(tempPath);
 
     // First create a 1 doc index: