LUCENE-5673: MMapDirectory: Work around a "bug" in the JDK that throws a confusing OutOfMemoryError wrapped inside IOException if the FileChannel mapping failed because of lack of virtual address space. The IOException is rethrown with more useful information about the problem, omitting the incorrect OutOfMemoryError

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1595213 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -188,6 +188,12 @@ Bug fixes
 
 * LUCENE-5668: Fix off-by-one in TieredMergePolicy (Mike McCandless)
 
+* LUCENE-5673: MMapDirectory: Work around a "bug" in the JDK that throws
+  a confusing OutOfMemoryError wrapped inside IOException if the FileChannel
+  mapping failed because of lack of virtual address space. The IOException is
+  rethrown with more useful information about the problem, omitting the
+  incorrect OutOfMemoryError.  (Robert Muir, Uwe Schindler)
+
 Test Framework
 
 * LUCENE-5622: Fail tests if they print over the given limit of bytes to 

lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java

@@ -28,6 +28,7 @@ import java.nio.file.StandardOpenOption;
 import java.security.AccessController;
 import java.security.PrivilegedExceptionAction;
 import java.security.PrivilegedActionException;
+import java.util.Locale;
 import java.lang.reflect.Method;
 
 import org.apache.lucene.util.Constants;
@@ -75,6 +76,7 @@ import org.apache.lucene.util.Constants;
  * blocked on IO. The channel will remain closed and subsequent access
  * to {@link MMapDirectory} will throw a {@link ClosedChannelException}. 
  * </p>
+ * @see <a href="http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html">Blog post about MMapDirectory</a>
  */
 public class MMapDirectory extends FSDirectory {
   private boolean useUnmapHack = UNMAP_SUPPORTED;
@@ -216,7 +218,7 @@ public class MMapDirectory extends FSDirectory {
     private final boolean useUnmapHack;
     
     MMapIndexInput(String resourceDescription, FileChannel fc) throws IOException {
-      super(resourceDescription, map(fc, 0, fc.size()), fc.size(), chunkSizePower, getUseUnmap());
+      super(resourceDescription, map(resourceDescription, fc, 0, fc.size()), fc.size(), chunkSizePower, getUseUnmap());
       this.useUnmapHack = getUseUnmap();
     }
     
@@ -244,18 +246,16 @@ public class MMapDirectory extends FSDirectory {
             }
           });
         } catch (PrivilegedActionException e) {
-          final IOException ioe = new IOException("unable to unmap the mapped buffer");
-          ioe.initCause(e.getCause());
-          throw ioe;
+          throw new IOException("Unable to unmap the mapped buffer: " + toString(), e.getCause());
         }
       }
     }
   }
   
   /** Maps a file into a set of buffers */
-  ByteBuffer[] map(FileChannel fc, long offset, long length) throws IOException {
+  final ByteBuffer[] map(String resourceDescription, FileChannel fc, long offset, long length) throws IOException {
     if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
-      throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + fc.toString());
+      throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + resourceDescription);
     
     final long chunkSize = 1L << chunkSizePower;
     
@@ -270,10 +270,45 @@ public class MMapDirectory extends FSDirectory {
           ? chunkSize
               : (length - bufferStart)
           );
-      buffers[bufNr] = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
+      try {
+        buffers[bufNr] = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
+      } catch (IOException ioe) {
+        throw convertMapFailedIOException(ioe, resourceDescription, bufSize);
+      }
       bufferStart += bufSize;
     }
     
     return buffers;
   }
+  
+  private IOException convertMapFailedIOException(IOException ioe, String resourceDescription, int bufSize) {
+    final String originalMessage;
+    final Throwable originalCause;
+    if (ioe.getCause() instanceof OutOfMemoryError) {
+      // nested OOM confuses users, because its "incorrect", just print a plain message:
+      originalMessage = "Map failed";
+      originalCause = null;
+    } else {
+      originalMessage = ioe.getMessage();
+      originalCause = ioe.getCause();
+    }
+    final String moreInfo;
+    if (!Constants.JRE_IS_64BIT) {
+      moreInfo = "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. ";
+    } else if (Constants.WINDOWS) {
+      moreInfo = "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. ";
+    } else if (Constants.LINUX) {
+      moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. ";
+    } else {
+      moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). ";
+    }
+    final IOException newIoe = new IOException(String.format(Locale.ENGLISH,
+        "%s: %s [this may be caused by lack of enough unfragmented virtual address space "+
+        "or too restrictive virtual memory limits enforced by the operating system, "+
+        "preventing us to map a chunk of %d bytes. %sMore information: "+
+        "http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]",
+        originalMessage, resourceDescription, bufSize, moreInfo), originalCause);
+    newIoe.setStackTrace(ioe.getStackTrace());
+    return newIoe;
+  }
 }