LUCENE-6072: add a way to test for too many open files

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1641821 13f79535-47bb-0310-9956-ffa450edef68

lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java

@@ -17,6 +17,7 @@ package org.apache.lucene.mockfile;
  * limitations under the License.
  */
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -28,9 +29,12 @@ import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -206,4 +210,29 @@ public class TestMockFilesystems extends LuceneTestCase {
     assertTrue(seenMessage.get());
     file.close();
   }
+  
+  public void testTooManyOpenFiles() throws IOException {
+    int n = 60;
+
+    Path dir = FilterPath.unwrap(createTempDir());
+    FileSystem fs = new HandleLimitFS(dir.getFileSystem(), n).getFileSystem(URI.create("file:///"));
+    dir = new FilterPath(dir, fs);
+    
+    // create open files to exact limit
+    List<Closeable> toClose = new ArrayList<>();
+    for (int i = 0; i < n; i++) {
+      Path p = Files.createTempFile(dir, null, null);
+      toClose.add(Files.newOutputStream(p));
+    }
+    
+    // now exceed
+    try {
+      Files.newOutputStream(Files.createTempFile(dir, null, null));
+      fail("didn't hit exception");
+    } catch (IOException e) {
+      assertTrue(e.getMessage().contains("Too many open files"));
+    }
+    
+    IOUtils.close(toClose);
+  }
 }

lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleLimitFS.java

@@ -0,0 +1,57 @@
+package org.apache.lucene.mockfile;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystemException;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/** 
+ * FileSystem that throws exception if file handles 
+ * in use exceeds a specified limit 
+ */
+public class HandleLimitFS extends HandleTrackingFS {
+  final int limit;
+  final AtomicInteger count = new AtomicInteger();
+  
+  /**
+   * Create a new instance, limiting the maximum number
+   * of open files to {@code limit}
+   * @param delegate delegate filesystem to wrap.
+   * @param limit maximum number of open files.
+   */
+  public HandleLimitFS(FileSystem delegate, int limit) {
+    super("handlelimit://", delegate);
+    this.limit = limit;
+  }
+
+  @Override
+  protected void onOpen(Path path, Object stream) throws IOException {
+    if (count.incrementAndGet() > limit) {
+      count.decrementAndGet();
+      throw new FileSystemException(path.toString(), null, "Too many open files");
+    }
+  }
+
+  @Override
+  protected void onClose(Path path, Object stream) throws IOException {
+    count.decrementAndGet();
+  }
+}

lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java

@@ -17,6 +17,7 @@ package org.apache.lucene.mockfile;
  * limitations under the License.
  */
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -34,6 +35,8 @@ import java.nio.file.attribute.FileAttribute;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 
+import org.apache.lucene.util.IOUtils;
+
 /** 
  * Base class for tracking file handles.
  * <p>
@@ -63,7 +66,6 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
    * @throws IOException if an I/O error occurs.
    */
   protected abstract void onOpen(Path path, Object stream) throws IOException;
-
   
   /**
    * Called when {@code path} is closed via {@code stream}. 
@@ -73,6 +75,21 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
    */
   protected abstract void onClose(Path path, Object stream) throws IOException;
 
+  /**
+   * Helper method, to deal with onOpen() throwing exception
+   */
+  final void callOpenHook(Path path, Closeable stream) throws IOException {
+    boolean success = false;
+    try {
+      onOpen(path, stream);
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeWhileHandlingException(stream);
+      }
+    }
+  }
+  
   @Override
   public InputStream newInputStream(Path path, OpenOption... options) throws IOException {
     InputStream stream = new FilterInputStream2(super.newInputStream(path, options)) {
@@ -97,7 +114,7 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
         return this == obj;
       }
     };
-    onOpen(path, stream);
+    callOpenHook(path, stream);
     return stream;
   }
 
@@ -125,7 +142,7 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
         return this == obj;
       }
     };
-    onOpen(path, stream);
+    callOpenHook(path, stream);
     return stream;
   }
   
@@ -153,7 +170,7 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
         return this == obj;
       }
     };
-    onOpen(path, channel);
+    callOpenHook(path, channel);
     return channel;
   }
 
@@ -181,7 +198,7 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
         return this == obj;
       }
     };
-    onOpen(path, channel);
+    callOpenHook(path, channel);
     return channel;
   }
 
@@ -209,7 +226,7 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
         return this == obj;
       }
     };
-    onOpen(path, channel);
+    callOpenHook(path, channel);
     return channel;
   }
 
@@ -242,7 +259,7 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
         }
       };
     }
-    onOpen(dir, stream);
+    callOpenHook(dir, stream);
     return stream;
   }
   
@@ -279,7 +296,7 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
     @Override
     public SecureDirectoryStream<Path> newDirectoryStream(Path path, LinkOption... options) throws IOException {
       SecureDirectoryStream<Path> stream = new TrackingSecureDirectoryStream(super.newDirectoryStream(path, options), path);
-      onOpen(path, stream);
+      callOpenHook(path, stream);
       return stream;
     }
 
@@ -307,7 +324,7 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
           return this == obj;
         }
       };
-      onOpen(path, channel);
+      callOpenHook(path, channel);
       return channel;
     }
   }

lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java

@@ -13,6 +13,7 @@ import java.util.Locale;
 import java.util.Random;
 
 import org.apache.lucene.mockfile.DisableFsyncFS;
+import org.apache.lucene.mockfile.HandleLimitFS;
 import org.apache.lucene.mockfile.LeakFS;
 import org.apache.lucene.mockfile.VerboseFS;
 import org.apache.lucene.mockfile.WindowsFS;
@@ -106,6 +107,10 @@ final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter {
     javaTempDir = initializeJavaTempDir();
   }
   
+  // os/config-independent limit for too many open files
+  // TODO: can we make this lower?
+  private static final int MAX_OPEN_FILES = 2048;
+  
   private FileSystem initializeFileSystem() {
     FileSystem fs = FileSystems.getDefault();
     if (LuceneTestCase.VERBOSE) {
@@ -116,6 +121,7 @@ final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter {
     if (random.nextInt(10) > 0) {
       fs = new DisableFsyncFS(fs).getFileSystem(null);
       fs = new LeakFS(fs).getFileSystem(null);
+      fs = new HandleLimitFS(fs, MAX_OPEN_FILES).getFileSystem(null);
       // windows is currently slow
       if (random.nextInt(10) == 0) {
         fs = new WindowsFS(fs).getFileSystem(null);