LUCENE-7345: make sure MDW's efforts to corrumpt un-sync'd files is successful

lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java

@@ -88,6 +88,9 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
   private ThrottledIndexOutput throttledOutput;
   private Throttling throttling = LuceneTestCase.TEST_NIGHTLY ? Throttling.SOMETIMES : Throttling.NEVER;
 
+  // for testing
+  boolean alwaysCorrupt;
+
   final AtomicInteger inputCloneCount = new AtomicInteger();
 
   // use this for tracking files for crash.
@@ -297,6 +300,9 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
     CollectionUtil.timSort(filesToCorrupt);
     for(String name : filesToCorrupt) {
       int damage = randomState.nextInt(6);
+      if (alwaysCorrupt && damage == 3) {
+        damage = 4;
+      }
       String action = null;
 
       switch(damage) {
@@ -313,9 +319,12 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
         try {
           length = fileLength(name);
         } catch (IOException ioe) {
-          // Ignore
+          throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
-          continue;
         }
+
+        // Delete original and write zeros back:
+        deleteFile(name);
+        
         byte[] zeroes = new byte[256];
         long upto = 0;
         try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
@@ -325,7 +334,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
             upto += limit;
           }
         } catch (IOException ioe) {
-          // ignore
+          throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
         }
         break;
 
@@ -341,8 +350,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
                IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState))) {
               tempFileName = tempOut.getName();
               tempOut.copyBytes(ii, ii.length()/2);
-            } catch (IOException ioe) {
+          } catch (IOException ioe) {
-            // ignore
+            throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
           }
 
           // Delete original and copy bytes back:
@@ -351,8 +360,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
           try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
                IndexInput ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState))) {
               out.copyBytes(ii, ii.length());
-            } catch (IOException ioe) {
+          } catch (IOException ioe) {
-            // ignore
+            throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
           }
           deleteFile(tempFileName);
         }
@@ -371,32 +380,32 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
           String tempFileName = null;
           try (IndexOutput tempOut = in.createTempOutput("name", "mdw_corrupt", LuceneTestCase.newIOContext(randomState));
                IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState))) {
-              tempFileName = tempOut.getName();
+            tempFileName = tempOut.getName();
-              if (ii.length() > 0) {
+            if (ii.length() > 0) {
-                // Copy first part unchanged:
+              // Copy first part unchanged:
-                long byteToCorrupt = (long) (randomState.nextDouble() * ii.length());
+              long byteToCorrupt = (long) (randomState.nextDouble() * ii.length());
-                if (byteToCorrupt > 0) {
+              if (byteToCorrupt > 0) {
-                  tempOut.copyBytes(ii, byteToCorrupt);
+                tempOut.copyBytes(ii, byteToCorrupt);
-                }
-
-                // Randomly flip one bit from this byte:
-                byte b = ii.readByte();
-                int bitToFlip = randomState.nextInt(8);
-                b = (byte) (b ^ (1 << bitToFlip));
-                tempOut.writeByte(b);
-
-                action = "flip bit " + bitToFlip + " of byte " + byteToCorrupt + " out of " + ii.length() + " bytes";
-
-                // Copy last part unchanged:
-                long bytesLeft = ii.length() - byteToCorrupt - 1;
-                if (bytesLeft > 0) {
-                  tempOut.copyBytes(ii, bytesLeft);
-                }
-              } else {
-                action = "didn't change";
               }
-            } catch (IOException ioe) {
+
-            // ignore
+              // Randomly flip one bit from this byte:
+              byte b = ii.readByte();
+              int bitToFlip = randomState.nextInt(8);
+              b = (byte) (b ^ (1 << bitToFlip));
+              tempOut.writeByte(b);
+
+              action = "flip bit " + bitToFlip + " of byte " + byteToCorrupt + " out of " + ii.length() + " bytes";
+
+              // Copy last part unchanged:
+              long bytesLeft = ii.length() - byteToCorrupt - 1;
+              if (bytesLeft > 0) {
+                tempOut.copyBytes(ii, bytesLeft);
+              }
+            } else {
+              action = "didn't change";
+            }
+          } catch (IOException ioe) {
+            throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
           }
 
           // Delete original and copy bytes back:
@@ -404,9 +413,9 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
 
           try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
                IndexInput ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState))) {
-              out.copyBytes(ii, ii.length());
+            out.copyBytes(ii, ii.length());
-            } catch (IOException ioe) {
+          } catch (IOException ioe) {
-            // ignore
+            throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
           }
 
           deleteFile(tempFileName);
@@ -421,7 +430,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
         try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
           out.getFilePointer(); // just fake access to prevent compiler warning
         } catch (IOException ioe) {
-          // ignore
+          throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
         }
         break;
 

lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java

@@ -16,7 +16,10 @@
  */
 package org.apache.lucene.store;
 
+import java.io.EOFException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 
 import org.apache.lucene.document.Document;
@@ -90,5 +93,82 @@ public class TestMockDirectoryWrapper extends BaseDirectoryTestCase {
     iw.commit();
     iw.close();
     dir.close();
-  }  
+  }
+
+  // just shields the wrapped directory from being closed
+  private static class PreventCloseDirectoryWrapper extends FilterDirectory {
+    public PreventCloseDirectoryWrapper(Directory in) {
+      super(in);
+    }
+
+    @Override
+    public void close() {
+    }
+  }
+
+  public void testCorruptOnCloseIsWorkingFSDir() throws Exception {
+    Path path = createTempDir();
+    try(Directory dir = newFSDirectory(path)) {
+      testCorruptOnCloseIsWorking(dir);
+    }
+  }
+
+  public void testCorruptOnCloseIsWorkingRAMDir() throws Exception {
+    try(Directory dir = new RAMDirectory()) {
+      testCorruptOnCloseIsWorking(dir);
+    }
+  }
+    
+  private void testCorruptOnCloseIsWorking(Directory dir) throws Exception {
+
+    dir = new PreventCloseDirectoryWrapper(dir);
+
+    try (MockDirectoryWrapper wrapped = new MockDirectoryWrapper(random(), dir)) {
+
+      // otherwise MDW sometimes randomly leaves the file intact and we'll see false test failures:
+      wrapped.alwaysCorrupt = true;
+
+      // MDW will only try to corrupt things if it sees an index:
+      RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+      iw.addDocument(new Document());
+      iw.close();
+      
+      // not sync'd!
+      try (IndexOutput out = wrapped.createOutput("foo", IOContext.DEFAULT)) {
+        for(int i=0;i<100;i++) {
+          out.writeInt(i);
+        }
+      }
+
+      // MDW.close now corrupts our unsync'd file (foo):
+    }
+
+    boolean changed = false;
+    IndexInput in = null;
+    try {
+      in = dir.openInput("foo", IOContext.DEFAULT);
+    } catch (NoSuchFileException | FileNotFoundException fnfe) {
+      // ok
+      changed = true;
+    }
+    if (in != null) {
+      for(int i=0;i<100;i++) {
+        int x;
+        try {
+          x = in.readInt();
+        } catch (EOFException eofe) {
+          changed = true;
+          break;
+        }
+        if (x != i) {
+          changed = true;
+          break;
+        }
+      }
+
+      in.close();
+    }
+
+    assertTrue("MockDirectoryWrapper on dir=" + dir + " failed to corrupt an unsync'd file", changed);
+  }
 }