fix a few tests; add some nocommits

lucene/core/src/java/org/apache/lucene/store/Directory.java

@@ -47,6 +47,7 @@ public abstract class Directory implements Closeable {
    * 
    * @throws IOException in case of IO error
    */
+  // nocommit should this sort?
   public abstract String[] listAll() throws IOException;
 
   /** Removes the specified files from the directory.  If an exception is thrown, behavior is undefined

lucene/core/src/java/org/apache/lucene/store/FSDirectory.java

@@ -229,22 +229,20 @@ public abstract class FSDirectory extends BaseDirectory {
     return listAll(directory, pendingDeletes);
   }
 
-  /** Returns the length in bytes of a file in the directory. */
   @Override
   public long fileLength(String name) throws IOException {
     ensureOpen();
     return Files.size(directory.resolve(name));
   }
 
-  /** Removes an existing file in the directory. */
   @Override
   public void deleteFiles(Collection<String> names) throws IOException {
     ensureOpen();
+    // nocommit isn't it an error if they were already pending delete?
     pendingDeletes.addAll(names);
     deletePendingFiles();
   }
 
-  /** Creates an IndexOutput for the file with the given name. */
   @Override
   public IndexOutput createOutput(String name, IOContext context) throws IOException {
     ensureOpen();
@@ -277,7 +275,7 @@ public abstract class FSDirectory extends BaseDirectory {
   protected void ensureCanRead(String name) throws IOException {
     deletePendingFiles();
     if (pendingDeletes.contains(name)) {
-      throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be overwritten");
+      throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be opened for read");
     }
   }
 
@@ -299,7 +297,6 @@ public abstract class FSDirectory extends BaseDirectory {
     IOUtils.fsync(directory, true);
   }
 
-  /** Closes the store to future operations. */
   @Override
   public synchronized void close() throws IOException {
     isOpen = false;
@@ -312,7 +309,6 @@ public abstract class FSDirectory extends BaseDirectory {
     return directory;
   }
 
-  /** For debug output. */
   @Override
   public String toString() {
     return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + lockFactory;
@@ -324,14 +320,13 @@ public abstract class FSDirectory extends BaseDirectory {
   }
 
   /** Returns true if the file was successfully removed. */
-  private boolean deleteFile(String name) throws IOException {  
+  private synchronized boolean deleteFile(String name) throws IOException {  
+    pendingDeletes.remove(name);
     try {
       Files.delete(directory.resolve(name));
-      pendingDeletes.remove(name);
       return true;
     } catch (NoSuchFileException | FileNotFoundException e) {
       // We were asked to delete a non-existent file:
-      pendingDeletes.remove(name);
       throw e;
     } catch (IOException ioe) {
       // On windows, a file delete can fail because there's still an open
@@ -358,22 +353,37 @@ public abstract class FSDirectory extends BaseDirectory {
   }
 
   /** Try to delete any pending files that we had previously tried to delete but failed
-   *  because we are on Windows and the files were still
-   *  held open. */
-  public void deletePendingFiles() throws IOException {
+   *  because we are on Windows and the files were still held open. */
+  public synchronized void deletePendingFiles() throws IOException {
     // TODO: we could fix IndexInputs from FSDirectory subclasses to call this when they are closed?
 
     // Clone the set because it will change as we iterate:
     List<String> toDelete = new ArrayList<>(pendingDeletes);
+    System.out.println("del pending: " + pendingDeletes);
 
     // First pass: delete any segments_N files.  We do these first to be certain stale commit points are removed
     // before we remove any files they reference.  If any delete of segments_N fails, we leave all other files
     // undeleted so index is never in a corrupt state:
+    Throwable firstException = null;
     for (String fileName : toDelete) {
       if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+        try {
           if (deleteFile(fileName) == false) {
+            // nocommit
+            System.out.println("  false on " + fileName + "; skipping the rest");
             return;
           }
+        } catch (Throwable t) {
+          if (firstException == null) {
+            firstException = t;
+          } else {
+            firstException.addSuppressed(t);
+          }
+          // nocommit
+          System.out.println("  fail on " + fileName + ":");
+          t.printStackTrace(System.out);
+          throw t;
+        }
       }
     }
 
@@ -381,11 +391,26 @@ public abstract class FSDirectory extends BaseDirectory {
     // leave a corrupt commit in the index even in the presense of virus checkers:
     for(String fileName : toDelete) {
       if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) {
+        try {
           deleteFile(fileName);
+        } catch (Throwable t) {
+          if (firstException == null) {
+            firstException = t;
+          } else {
+            firstException.addSuppressed(t);
+          }
+          // nocommit
+          System.out.println("  fail on " + fileName + ":");
+          t.printStackTrace(System.out);
+          throw t;
         }
       }
     }
 
+    // Does nothing if firstException is null:
+    IOUtils.reThrow(firstException);
+  }
+
   final class FSIndexOutput extends OutputStreamIndexOutput {
     /**
      * The maximum chunk size is 8192 bytes, because file channel mallocs

lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java

@@ -472,7 +472,10 @@ void assertTermDocsCount(String msg,
   public void testOpenReaderAfterDelete() throws IOException {
     Path dirFile = createTempDir("deletetest");
     Directory dir = newFSDirectory(dirFile);
-    assumeFalse("test deletes files directly", TestUtil.hasVirusChecker(dir));
+    if (TestUtil.hasVirusChecker(dir)) {
+      dir.close();
+      assumeTrue("test deletes files directly", false);
+    }
     if (dir instanceof BaseDirectoryWrapper) {
       ((BaseDirectoryWrapper)dir).setCheckIndexOnClose(false); // we will hit NoSuchFileException in MDW since we nuked it!
     }
@@ -1055,7 +1058,6 @@ void assertTermDocsCount(String msg,
 
   public void testIndexExistsOnNonExistentDirectory() throws Exception {
     Path tempDir = createTempDir("testIndexExistsOnNonExistentDirectory");
-    IOUtils.rm(tempDir);
     Directory dir = newFSDirectory(tempDir);
     assertFalse(DirectoryReader.indexExists(dir));
     dir.close();

lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java

@@ -38,8 +38,11 @@ import java.util.UUID;
 import org.apache.lucene.mockfile.FilterFileSystem;
 import org.apache.lucene.mockfile.FilterFileSystemProvider;
 import org.apache.lucene.mockfile.FilterPath;
+import org.apache.lucene.mockfile.VirusCheckingFS;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
 
 /** Simple test methods for IOUtils */
+@SuppressFileSystems("VirusCheckingFS")
 public class TestIOUtils extends LuceneTestCase {
   
   public void testDeleteFileIgnoringExceptions() throws Exception {

lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java

@@ -25,6 +25,7 @@ import java.nio.file.Path;
 import java.util.Random;
 
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.util.LuceneTestCase;
 
 /** 
  * Acts like Windows, where random programs may open the files you just wrote in an unfriendly
@@ -56,6 +57,9 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
         && Files.exists(path) // important that we NOT delay a NoSuchFileException until later
         && path.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME) == false // life is particularly difficult if the virus checker hits our lock file
         && random.nextInt(5) == 1) {
+      if (true || LuceneTestCase.VERBOSE) {
+        System.out.println("NOTE: VirusCheckingFS now refusing to delete " + path);
+      }
       throw new AccessDeniedException("VirusCheckingFS is randomly refusing to delete file \"" + path + "\"");
     }
     super.delete(path);