Merge pull request #11433 from rmuir/cleanup_mlockall

Better error messages when mlockall fails

src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -84,14 +84,6 @@ public class Bootstrap {
     /** initialize native resources */
     public static void initializeNatives(boolean mlockAll, boolean ctrlHandler, boolean loadSigar) {
         final ESLogger logger = Loggers.getLogger(Bootstrap.class);
-        // mlockall if requested
-        if (mlockAll) {
-            if (Constants.WINDOWS) {
-               Natives.tryVirtualLock();
-            } else {
-               Natives.tryMlockall();
-            }
-        }
         
         // check if the user is running as root, and bail
         if (Natives.definitelyRunningAsRoot()) {
@@ -101,6 +93,15 @@ public class Bootstrap {
                 throw new RuntimeException("don't run elasticsearch as root.");
             }
         }
+        
+        // mlockall if requested
+        if (mlockAll) {
+            if (Constants.WINDOWS) {
+               Natives.tryVirtualLock();
+            } else {
+               Natives.tryMlockall();
+            }
+        }
 
         // listener for windows close event
         if (ctrlHandler) {

src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java

@@ -20,33 +20,53 @@
 package org.elasticsearch.bootstrap;
 
 import com.sun.jna.Native;
+import com.sun.jna.Structure;
+
+import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 
+import java.util.Arrays;
+import java.util.List;
 
 /**
- *
+ * java mapping to some libc functions
  */
-class JNACLibrary {
+final class JNACLibrary {
 
     private static final ESLogger logger = Loggers.getLogger(JNACLibrary.class);
 
     public static final int MCL_CURRENT = 1;
-    public static final int MCL_FUTURE = 2;
-
     public static final int ENOMEM = 12;
+    public static final int RLIMIT_MEMLOCK = Constants.MAC_OS_X ? 6 : 8;
+    public static final long RLIM_INFINITY = Constants.MAC_OS_X ? 9223372036854775807L : -1L;
 
     static {
         try {
             Native.register("c");
         } catch (UnsatisfiedLinkError e) {
-            logger.warn("unable to link C library. native methods (mlockall) will be disabled.");
+            logger.warn("unable to link C library. native methods (mlockall) will be disabled.", e);
         }
     }
 
     static native int mlockall(int flags);
 
     static native int geteuid();
+    
+    /** corresponds to struct rlimit */
+    public static final class Rlimit extends Structure implements Structure.ByReference {
+        public long rlim_cur = 0;
+        public long rlim_max = 0;
+        
+        @Override
+        protected List getFieldOrder() {
+            return Arrays.asList(new String[] { "rlim_cur", "rlim_max" });
+        }
+    }
+    
+    static native int getrlimit(int resource, Rlimit rlimit);
+    
+    static native String strerror(int errno);
 
     private JNACLibrary() {
     }

src/main/java/org/elasticsearch/bootstrap/JNANatives.java

@@ -26,8 +26,6 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
-import java.util.Locale;
-
 import static org.elasticsearch.bootstrap.JNAKernel32Library.SizeT;
 
 /**
@@ -43,30 +41,66 @@ class JNANatives {
 
     static void tryMlockall() {
         int errno = Integer.MIN_VALUE;
+        String errMsg = null;
+        boolean rlimitSuccess = false;
+        long softLimit = 0;
+        long hardLimit = 0;
+        
         try {
             int result = JNACLibrary.mlockall(JNACLibrary.MCL_CURRENT);
-            if (result != 0) {
+            if (result == 0) {
-                errno = Native.getLastError();
-            } else {
                 LOCAL_MLOCKALL = true;
+                return;
+            }
+            
+            errno = Native.getLastError();
+            errMsg = JNACLibrary.strerror(errno);
+            if (Constants.LINUX || Constants.MAC_OS_X) {
+                // we only know RLIMIT_MEMLOCK for these two at the moment. 
+                JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit();
+                if (JNACLibrary.getrlimit(JNACLibrary.RLIMIT_MEMLOCK, rlimit) == 0) {
+                    rlimitSuccess = true;
+                    softLimit = rlimit.rlim_cur;
+                    hardLimit = rlimit.rlim_max;
+                } else {
+                    logger.warn("Unable to retrieve resource limits: " + JNACLibrary.strerror(Native.getLastError()));
+                }
             }
         } catch (UnsatisfiedLinkError e) {
             // this will have already been logged by CLibrary, no need to repeat it
             return;
         }
 
-        if (errno != Integer.MIN_VALUE) {
+        // mlockall failed for some reason
-            if (errno == JNACLibrary.ENOMEM && System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("linux")) {
+        logger.warn("Unable to lock JVM Memory: error=" + errno + ",reason=" + errMsg + ". This can result in part of the JVM being swapped out.");
-                logger.warn("Unable to lock JVM memory (ENOMEM)."
+        if (errno == JNACLibrary.ENOMEM) {
-                        + " This can result in part of the JVM being swapped out."
+            if (rlimitSuccess) {
-                        + " Increase RLIMIT_MEMLOCK (ulimit).");
+                logger.warn("Increase RLIMIT_MEMLOCK, soft limit: " + rlimitToString(softLimit) + ", hard limit: " + rlimitToString(hardLimit));
-            } else if (!System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("mac")) {
+                if (Constants.LINUX) {
-                // OS X allows mlockall to be called, but always returns an error
+                    // give specific instructions for the linux case to make it easy
-                logger.warn("Unknown mlockall error " + errno);
+                    logger.warn("These can be adjusted by modifying /etc/security/limits.conf, for example: \n" +
+                                "\t# allow user 'esuser' mlockall\n" +
+                                "\tesuser soft memlock unlimited\n" +
+                                "\tesuser hard memlock unlimited"
+                               );
+                    logger.warn("If you are logged in interactively, you will have to re-login for the new limits to take effect.");
+                }
+            } else {
+                logger.warn("Increase RLIMIT_MEMLOCK (ulimit).");
             }
         }
     }
     
+    static String rlimitToString(long value) {
+        assert Constants.LINUX || Constants.MAC_OS_X;
+        if (value == JNACLibrary.RLIM_INFINITY) {
+            return "unlimited";
+        } else {
+            // TODO, on java 8 use Long.toUnsignedString, since thats what it is.
+            return Long.toString(value);
+        }
+    }
+
     /** Returns true if user is root, false if not, or if we don't know */
     static boolean definitelyRunningAsRoot() {
         if (Constants.WINDOWS) {