Native (java) process memory leak, closes #1118.

modules/elasticsearch/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java

@@ -61,9 +61,11 @@ public class JvmMonitorService extends AbstractLifecycleComponent<JvmMonitorServ
         this.threadPool = threadPool;
         this.dumpMonitorService = dumpMonitorService;
 
-        this.enabled = componentSettings.getAsBoolean("enabled", true);
+        this.enabled = componentSettings.getAsBoolean("enabled", JvmStats.isLastGcEnabled());
         this.interval = componentSettings.getAsTime("interval", timeValueSeconds(1));
         this.gcThreshold = componentSettings.getAsTime("gc_threshold", timeValueMillis(5000));
+
+        logger.debug("enabled [{}], last_gc_enabled [{}], interval [{}], gc_threshold [{}]", enabled, JvmStats.isLastGcEnabled(), interval, gcThreshold);
     }
 
     @Override protected void doStart() throws ElasticSearchException {

modules/elasticsearch/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.monitor.jvm;
 
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -47,11 +48,16 @@ import java.util.concurrent.TimeUnit;
  */
 public class JvmStats implements Streamable, Serializable, ToXContent {
 
+    private static boolean enableLastGc;
+
+    public static boolean isLastGcEnabled() {
+        return enableLastGc;
+    }
+
     private final static RuntimeMXBean runtimeMXBean;
     private final static MemoryMXBean memoryMXBean;
     private final static ThreadMXBean threadMXBean;
 
-    private static boolean sunGc;
     private static Method getLastGcInfoMethod;
     private static Method getMemoryUsageBeforeGcMethod;
     private static Method getMemoryUsageAfterGcMethod;
@@ -64,28 +70,32 @@ public class JvmStats implements Streamable, Serializable, ToXContent {
         memoryMXBean = ManagementFactory.getMemoryMXBean();
         threadMXBean = ManagementFactory.getThreadMXBean();
 
-        try {
+        boolean enableLastGc = Booleans.parseBoolean(System.getProperty("monitory.jvm.enable_last_gc"), false);
-            Class sunGcClass = Class.forName("com.sun.management.GarbageCollectorMXBean");
+        if (enableLastGc) {
-            Class gcInfoClass = Class.forName("com.sun.management.GcInfo");
+            try {
+                Class sunGcClass = Class.forName("com.sun.management.GarbageCollectorMXBean");
+                Class gcInfoClass = Class.forName("com.sun.management.GcInfo");
 
-            getLastGcInfoMethod = sunGcClass.getDeclaredMethod("getLastGcInfo");
+                getLastGcInfoMethod = sunGcClass.getDeclaredMethod("getLastGcInfo");
-            getLastGcInfoMethod.setAccessible(true);
+                getLastGcInfoMethod.setAccessible(true);
 
-            getMemoryUsageBeforeGcMethod = gcInfoClass.getDeclaredMethod("getMemoryUsageBeforeGc");
+                getMemoryUsageBeforeGcMethod = gcInfoClass.getDeclaredMethod("getMemoryUsageBeforeGc");
-            getMemoryUsageBeforeGcMethod.setAccessible(true);
+                getMemoryUsageBeforeGcMethod.setAccessible(true);
-            getMemoryUsageAfterGcMethod = gcInfoClass.getDeclaredMethod("getMemoryUsageAfterGc");
+                getMemoryUsageAfterGcMethod = gcInfoClass.getDeclaredMethod("getMemoryUsageAfterGc");
-            getMemoryUsageAfterGcMethod.setAccessible(true);
+                getMemoryUsageAfterGcMethod.setAccessible(true);
-            getStartTimeMethod = gcInfoClass.getDeclaredMethod("getStartTime");
+                getStartTimeMethod = gcInfoClass.getDeclaredMethod("getStartTime");
-            getStartTimeMethod.setAccessible(true);
+                getStartTimeMethod.setAccessible(true);
-            getEndTimeMethod = gcInfoClass.getDeclaredMethod("getEndTime");
+                getEndTimeMethod = gcInfoClass.getDeclaredMethod("getEndTime");
-            getEndTimeMethod.setAccessible(true);
+                getEndTimeMethod.setAccessible(true);
-            getDurationMethod = gcInfoClass.getDeclaredMethod("getDuration");
+                getDurationMethod = gcInfoClass.getDeclaredMethod("getDuration");
-            getDurationMethod.setAccessible(true);
+                getDurationMethod.setAccessible(true);
 
-            sunGc = true;
+            } catch (Throwable ex) {
-        } catch (Throwable ex) {
+                enableLastGc = false;
-            sunGc = false;
+            }
         }
+
+        JvmStats.enableLastGc = false;
     }
 
     public static JvmStats jvmStats() {
@@ -111,7 +121,7 @@ public class JvmStats implements Streamable, Serializable, ToXContent {
             stats.gc.collectors[i].name = gcMxBean.getName();
             stats.gc.collectors[i].collectionCount = gcMxBean.getCollectionCount();
             stats.gc.collectors[i].collectionTime = gcMxBean.getCollectionTime();
-            if (sunGc) {
+            if (enableLastGc) {
                 try {
                     Object lastGcInfo = getLastGcInfoMethod.invoke(gcMxBean);
                     if (lastGcInfo != null) {

modules/test/integration/src/test/java/org/elasticsearch/test/stress/leaks/GenericStatsLeak.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.stress.leaks;
+
+import org.elasticsearch.monitor.jvm.JvmService;
+import org.elasticsearch.monitor.network.NetworkService;
+import org.elasticsearch.monitor.os.OsService;
+import org.elasticsearch.monitor.process.ProcessService;
+import org.elasticsearch.node.NodeBuilder;
+import org.elasticsearch.node.internal.InternalNode;
+
+public class GenericStatsLeak {
+
+    public static void main(String[] args) {
+        InternalNode node = (InternalNode) NodeBuilder.nodeBuilder().node();
+
+        JvmService jvmService = node.injector().getInstance(JvmService.class);
+        OsService osService = node.injector().getInstance(OsService.class);
+        ProcessService processService = node.injector().getInstance(ProcessService.class);
+        NetworkService networkService = node.injector().getInstance(NetworkService.class);
+
+        while (true) {
+            jvmService.stats();
+            osService.stats();
+            processService.stats();
+            networkService.stats();
+        }
+    }
+}

modules/test/integration/src/test/java/org/elasticsearch/test/stress/leaks/JvmStatsLeak.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.stress.leaks;
+
+import org.elasticsearch.monitor.jvm.JvmStats;
+
+/**
+ * This test mainly comes to check the native memory leak with getLastGCInfo (which is now
+ * disabled by default).
+ */
+public class JvmStatsLeak {
+
+    public static void main(String[] args) {
+        while (true) {
+            JvmStats.jvmStats();
+        }
+    }
+}