Issue #2775 LowResourceMonitor extendable (#2812)

* make LowResourceMonitor extendable #2775

Signed-off-by: olivier lamy <oliver.lamy@gmail.com>

jetty-server/src/main/config/etc/jetty-lowresources.xml

@@ -8,7 +8,7 @@
 <Configure id="Server" class="org.eclipse.jetty.server.Server">
   <Call name="addBean">
     <Arg>
-      <New class="org.eclipse.jetty.server.LowResourceMonitor">
+      <New id="lowResourceMonitor" class="org.eclipse.jetty.server.LowResourceMonitor">
         <Arg name="server"><Ref refid='Server'/></Arg>
         <Set name="period"><Property name="jetty.lowresources.period" deprecated="lowresources.period" default="1000"/></Set>
         <Set name="lowResourcesIdleTimeout"><Property name="jetty.lowresources.idleTimeout" deprecated="lowresources.lowResourcesIdleTimeout" default="1000"/></Set>
@@ -16,7 +16,7 @@
         <Set name="maxConnections"><Property name="jetty.lowresources.maxConnections" deprecated="lowresources.maxConnections" default="0"/></Set>
         <Set name="maxMemory"><Property name="jetty.lowresources.maxMemory" deprecated="lowresources.maxMemory" default="0"/></Set>
         <Set name="maxLowResourcesTime"><Property name="jetty.lowresources.maxLowResourcesTime" deprecated="lowresources.maxLowResourcesTime" default="5000"/></Set>
-	<Set name="acceptingInLowResources"><Property name="jetty.lowresources.accepting" default="true"/></Set>
+        <Set name="acceptingInLowResources"><Property name="jetty.lowresources.accepting" default="true"/></Set>
       </New>
     </Arg>
   </Call>

jetty-server/src/main/java/org/eclipse/jetty/server/LowResourceMonitor.java

@@ -18,6 +18,17 @@
 
 package org.eclipse.jetty.server;
 
+import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.util.annotation.ManagedAttribute;
+import org.eclipse.jetty.util.annotation.ManagedObject;
+import org.eclipse.jetty.util.annotation.Name;
+import org.eclipse.jetty.util.component.ContainerLifeCycle;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
+import org.eclipse.jetty.util.thread.Scheduler;
+import org.eclipse.jetty.util.thread.ThreadPool;
+
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -27,66 +38,46 @@ import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import org.eclipse.jetty.io.EndPoint;
-import org.eclipse.jetty.util.annotation.ManagedAttribute;
-import org.eclipse.jetty.util.annotation.ManagedObject;
-import org.eclipse.jetty.util.annotation.Name;
-import org.eclipse.jetty.util.component.AbstractLifeCycle;
-import org.eclipse.jetty.util.log.Log;
-import org.eclipse.jetty.util.log.Logger;
-import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
-import org.eclipse.jetty.util.thread.Scheduler;
-import org.eclipse.jetty.util.thread.ThreadPool;
-
 
 /**
- * A monitor for low resources
+ *
- * <p>
+ * A monitor for low resources, low resources can be detected by:
- * An instance of this class will monitor all the connectors of a server (or a set of connectors
- * configured with {@link #setMonitoredConnectors(Collection)}) for a low resources state.
- * <p>
- * Low resources can be detected by:
  * <ul>
  * <li>{@link ThreadPool#isLowOnThreads()} if {@link Connector#getExecutor()} is
  * an instance of {@link ThreadPool} and {@link #setMonitorThreads(boolean)} is true.</li>
  * <li>If {@link #setMaxMemory(long)} is non zero then low resources is detected if the JVMs
  * {@link Runtime} instance has {@link Runtime#totalMemory()} minus {@link Runtime#freeMemory()}
  * greater than {@link #getMaxMemory()}</li>
- * <li>If {@link #setMaxConnections(int)} is non zero then low resources is dected if the total number
+ * <li>If {@link #setMaxConnections(int)} is non zero then low resources is detected if the total number
  * of connections exceeds {@link #getMaxConnections()}.  This feature is deprecated and replaced by
  * {@link ConnectionLimit}</li>
  * </ul>
- * <p>
+ *
- * Once low resources state is detected, the cause is logged and all existing connections returned
- * by {@link Connector#getConnectedEndPoints()} have {@link EndPoint#setIdleTimeout(long)} set
- * to {@link #getLowResourcesIdleTimeout()}.  New connections are not affected, however if the low
- * resources state persists for more than {@link #getMaxLowResourcesTime()}, then the
- * {@link #getLowResourcesIdleTimeout()} to all connections again.  Once the low resources state is
- * cleared, the idle timeout is reset to the connector default given by {@link Connector#getIdleTimeout()}.
- * <p>
- * If {@link #setAcceptingInLowResources(boolean)} is set to false (Default is true), then no new connections 
- * are accepted when in low resources state.
  */
 @ManagedObject ("Monitor for low resource conditions and activate a low resource mode if detected")
-public class LowResourceMonitor extends AbstractLifeCycle
+public class LowResourceMonitor extends ContainerLifeCycle
 {
     private static final Logger LOG = Log.getLogger(LowResourceMonitor.class);
-    private final Server _server;
+
+    protected final Server _server;
     private Scheduler _scheduler;
     private Connector[] _monitoredConnectors;
     private Set<AbstractConnector> _acceptingConnectors = new HashSet<>();
     private int _period=1000;
-    private int _maxConnections;
+
-    private long _maxMemory;
+
     private int _lowResourcesIdleTimeout=1000;
     private int _maxLowResourcesTime=0;
-    private boolean _monitorThreads=true;
+
     private final AtomicBoolean _low = new AtomicBoolean();
-    private String _cause;
+
     private String _reasons;
+
     private long _lowStarted;
     private boolean _acceptingInLowResources = true;
 
+    private Set<LowResourceCheck> _lowResourceChecks = new HashSet<>();
+
     private final Runnable _monitor = new Runnable()
     {
         @Override
@@ -95,14 +86,86 @@ public class LowResourceMonitor extends AbstractLifeCycle
             if (isRunning())
             {
                 monitor();
-                _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
+                _scheduler.schedule( _monitor, _period, TimeUnit.MILLISECONDS);
             }
         }
     };
 
     public LowResourceMonitor(@Name("server") Server server)
     {
-        _server=server;
+        _server = server;
+    }
+
+    @ManagedAttribute("True if low available threads status is monitored")
+    public boolean getMonitorThreads()
+    {
+        return !getBeans(ConnectorsThreadPoolLowResourceCheck.class).isEmpty();
+    }
+
+    /**
+     * @param monitorThreads If true, check connectors executors to see if they are
+     * {@link ThreadPool} instances that are low on threads.
+     */
+    public void setMonitorThreads(boolean monitorThreads)
+    {
+        if(monitorThreads)
+            // already configured?
+            if ( !getMonitorThreads() ) addLowResourceCheck( new ConnectorsThreadPoolLowResourceCheck() );
+        else
+            getBeans(ConnectorsThreadPoolLowResourceCheck.class).forEach(this::removeBean);
+    }
+
+    /**
+     * @return The maximum connections allowed for the monitored connectors before low resource handling is activated
+     * @deprecated Replaced by ConnectionLimit
+     */
+    @ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated")
+    @Deprecated
+    public int getMaxConnections()
+    {
+        for(MaxConnectionsLowResourceCheck lowResourceCheck : getBeans(MaxConnectionsLowResourceCheck.class))
+        {
+            if (lowResourceCheck.getMaxConnections()>0)
+            {
+                return lowResourceCheck.getMaxConnections();
+            }
+        }
+        return -1;
+    }
+
+    /**
+     * @param maxConnections The maximum connections before low resources state is triggered
+     * @deprecated Replaced by ConnectionLimit
+     */
+    @Deprecated
+    public void setMaxConnections(int maxConnections)
+    {
+        if (maxConnections>0)
+        {
+            if (getBeans(MaxConnectionsLowResourceCheck.class).isEmpty())
+            {
+                addLowResourceCheck(new MaxConnectionsLowResourceCheck(maxConnections));
+            } else
+            {
+                getBeans(MaxConnectionsLowResourceCheck.class).forEach( c -> c.setMaxConnections( maxConnections ) );
+            }
+        }
+        else
+        {
+            getBeans(ConnectorsThreadPoolLowResourceCheck.class).forEach(this::removeBean);
+        }
+    }
+
+
+    @ManagedAttribute("The reasons the monitored connectors are low on resources")
+    public String getReasons()
+    {
+        return _reasons;
+    }
+
+    protected void setReasons(String reasons)
+    {
+        _reasons = reasons;
     }
 
     @ManagedAttribute("Are the monitored connectors low on resources?")
@@ -111,24 +174,39 @@ public class LowResourceMonitor extends AbstractLifeCycle
         return _low.get();
     }
 
+    protected boolean enableLowOnResources(boolean expectedValue, boolean newValue)
+    {
+        return _low.compareAndSet(expectedValue, newValue);
+    }
+
     @ManagedAttribute("The reason(s) the monitored connectors are low on resources")
     public String getLowResourcesReasons()
     {
         return _reasons;
     }
 
+    protected void setLowResourcesReasons(String reasons)
+    {
+        _reasons = reasons;
+    }
+
     @ManagedAttribute("Get the timestamp in ms since epoch that low resources state started")
     public long getLowResourcesStarted()
     {
         return _lowStarted;
     }
 
+    public void setLowResourcesStarted(long lowStarted)
+    {
+        _lowStarted = lowStarted;
+    }
+
     @ManagedAttribute("The monitored connectors. If null then all server connectors are monitored")
     public Collection<Connector> getMonitoredConnectors()
     {
         if (_monitoredConnectors==null)
             return Collections.emptyList();
-        return Arrays.asList(_monitoredConnectors);
+        return Arrays.asList( _monitoredConnectors);
     }
 
     /**
@@ -142,6 +220,13 @@ public class LowResourceMonitor extends AbstractLifeCycle
             _monitoredConnectors = monitoredConnectors.toArray(new Connector[monitoredConnectors.size()]);
     }
 
+    protected Connector[] getMonitoredOrServerConnectors()
+    {
+        if (_monitoredConnectors!=null && _monitoredConnectors.length>0)
+            return _monitoredConnectors;
+        return _server.getConnectors();
+    }
+
     @ManagedAttribute("If false, new connections are not accepted while in low resources")
     public boolean isAcceptingInLowResources()
     {
@@ -152,7 +237,7 @@ public class LowResourceMonitor extends AbstractLifeCycle
     {
         _acceptingInLowResources = acceptingInLowResources;
     }
-    
+
     @ManagedAttribute("The monitor period in ms")
     public int getPeriod()
     {
@@ -167,58 +252,6 @@ public class LowResourceMonitor extends AbstractLifeCycle
         _period = periodMS;
     }
 
-    @ManagedAttribute("True if low available threads status is monitored")
-    public boolean getMonitorThreads()
-    {
-        return _monitorThreads;
-    }
-
-    /**
-     * @param monitorThreads If true, check connectors executors to see if they are
-     * {@link ThreadPool} instances that are low on threads.
-     */
-    public void setMonitorThreads(boolean monitorThreads)
-    {
-        _monitorThreads = monitorThreads;
-    }
-
-    /**
-     * @return The maximum connections allowed for the monitored connectors before low resource handling is activated
-     * @deprecated Replaced by ConnectionLimit
-     */
-    @ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated")
-    @Deprecated
-    public int getMaxConnections()
-    {
-        return _maxConnections;
-    }
-
-    /**
-     * @param maxConnections The maximum connections before low resources state is triggered
-     * @deprecated Replaced by ConnectionLimit
-     */
-    @Deprecated
-    public void setMaxConnections(int maxConnections)
-    {
-        if (maxConnections>0)
-            LOG.warn("LowResourceMonitor.setMaxConnections is deprecated. Use ConnectionLimit.");
-        _maxConnections = maxConnections;
-    }
-
-    @ManagedAttribute("The maximum memory (in bytes) that can be used before low resources is triggered.  Memory used is calculated as (totalMemory-freeMemory).")
-    public long getMaxMemory()
-    {
-        return _maxMemory;
-    }
-
-    /**
-     * @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered.
-     */
-    public void setMaxMemory(long maxMemoryBytes)
-    {
-        _maxMemory = maxMemoryBytes;
-    }
-
     @ManagedAttribute("The idletimeout in ms to apply to all existing connections when low resources is detected")
     public int getLowResourcesIdleTimeout()
     {
@@ -247,6 +280,99 @@ public class LowResourceMonitor extends AbstractLifeCycle
         _maxLowResourcesTime = maxLowResourcesTimeMS;
     }
 
+    @ManagedAttribute("The maximum memory (in bytes) that can be used before low resources is triggered.  Memory used is calculated as (totalMemory-freeMemory).")
+    public long getMaxMemory()
+    {
+        Collection<MemoryLowResourceCheck> beans = getBeans(MemoryLowResourceCheck.class);
+        if(beans.isEmpty())
+        {
+            return 0;
+        }
+        return beans.stream().findFirst().get().getMaxMemory();
+    }
+
+    /**
+     * @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered.
+     */
+    public void setMaxMemory(long maxMemoryBytes)
+    {
+        if(maxMemoryBytes<=0)
+        {
+            return;
+        }
+        Collection<MemoryLowResourceCheck> beans = getBeans(MemoryLowResourceCheck.class);
+        if(beans.isEmpty())
+            addLowResourceCheck( new MemoryLowResourceCheck( maxMemoryBytes ) );
+        else
+            beans.forEach( lowResourceCheck -> lowResourceCheck.setMaxMemory( maxMemoryBytes ) );
+    }
+
+    public Set<LowResourceCheck> getLowResourceChecks()
+    {
+        return _lowResourceChecks;
+    }
+
+    public void setLowResourceChecks( Set<LowResourceCheck> lowResourceChecks )
+    {
+        updateBeans(_lowResourceChecks.toArray(),lowResourceChecks.toArray());
+        this._lowResourceChecks = lowResourceChecks;
+    }
+
+    public void addLowResourceCheck( LowResourceCheck lowResourceCheck )
+    {
+        addBean(lowResourceCheck);
+        this._lowResourceChecks.add(lowResourceCheck);
+    }
+
+    protected void monitor()
+    {
+
+        String reasons=null;
+
+
+        for(LowResourceCheck lowResourceCheck : _lowResourceChecks)
+        {
+            if(lowResourceCheck.isLowOnResources())
+            {
+                reasons = lowResourceCheck.toString();
+                break;
+            }
+        }
+
+        if (reasons!=null)
+        {
+            // Log the reasons if there is any change in the cause
+            if (!reasons.equals(getReasons()))
+            {
+                LOG.warn("Low Resources: {}",reasons);
+                setReasons(reasons);
+            }
+
+            // Enter low resources state?
+            if (enableLowOnResources(false,true))
+            {
+                setLowResourcesReasons(reasons);
+                setLowResourcesStarted(System.currentTimeMillis());
+                setLowResources();
+            }
+
+            // Too long in low resources state?
+            if ( getMaxLowResourcesTime()>0 && (System.currentTimeMillis()-getLowResourcesStarted())>getMaxLowResourcesTime())
+                setLowResources();
+        }
+        else
+        {
+            if (enableLowOnResources(true,false))
+            {
+                LOG.info("Low Resources cleared");
+                setLowResourcesReasons(null);
+                setLowResourcesStarted(0);
+                setReasons(null);
+                clearLowResources();
+            }
+        }
+    }
+
     @Override
     protected void doStart() throws Exception
     {
@@ -257,6 +383,7 @@ public class LowResourceMonitor extends AbstractLifeCycle
             _scheduler=new LRMScheduler();
             _scheduler.start();
         }
+
         super.doStart();
 
         _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
@@ -265,94 +392,11 @@ public class LowResourceMonitor extends AbstractLifeCycle
     @Override
     protected void doStop() throws Exception
     {
-        if (_scheduler instanceof LRMScheduler)
+        if (_scheduler instanceof LRMScheduler )
             _scheduler.stop();
         super.doStop();
     }
 
-    protected Connector[] getMonitoredOrServerConnectors()
-    {
-        if (_monitoredConnectors!=null && _monitoredConnectors.length>0)
-            return _monitoredConnectors;
-        return _server.getConnectors();
-    }
-
-    protected void monitor()
-    {
-        String reasons=null;
-        String cause="";
-        int connections=0;
-
-        ThreadPool serverThreads = _server.getThreadPool();
-        if (_monitorThreads && serverThreads.isLowOnThreads())
-        {
-            reasons=low(reasons,"Server low on threads: "+serverThreads);
-            cause+="S";
-        }
-
-        for(Connector connector : getMonitoredOrServerConnectors())
-        {
-            connections+=connector.getConnectedEndPoints().size();
-
-            Executor executor = connector.getExecutor();
-            if (executor instanceof ThreadPool && executor!=serverThreads)
-            {
-                ThreadPool connectorThreads=(ThreadPool)executor;
-                if (_monitorThreads && connectorThreads.isLowOnThreads())
-                {
-                    reasons=low(reasons,"Connector low on threads: "+connectorThreads);
-                    cause+="T";
-                }
-            }
-        }
-
-        if (_maxConnections>0 && connections>_maxConnections)
-        {
-            reasons=low(reasons,"Max Connections exceeded: "+connections+">"+_maxConnections);
-            cause+="C";
-        }
-
-        long memory=Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory();
-        if (_maxMemory>0 && memory>_maxMemory)
-        {
-            reasons=low(reasons,"Max memory exceeded: "+memory+">"+_maxMemory);
-            cause+="M";
-        }
-
-        if (reasons!=null)
-        {
-            // Log the reasons if there is any change in the cause
-            if (!cause.equals(_cause))
-            {
-                LOG.warn("Low Resources: {}",reasons);
-                _cause=cause;
-            }
-
-            // Enter low resources state?
-            if (_low.compareAndSet(false,true))
-            {
-                _reasons=reasons;
-                _lowStarted=System.currentTimeMillis();
-                setLowResources();
-            }
-
-            // Too long in low resources state?
-            if (_maxLowResourcesTime>0 && (System.currentTimeMillis()-_lowStarted)>_maxLowResourcesTime)
-                setLowResources();
-        }
-        else
-        {
-            if (_low.compareAndSet(true,false))
-            {
-                LOG.info("Low Resources cleared");
-                _reasons=null;
-                _lowStarted=0;
-                _cause=null;
-                clearLowResources();
-            }
-        }
-    }
-
     protected void setLowResources()
     {
         for(Connector connector : getMonitoredOrServerConnectors())
@@ -366,8 +410,8 @@ public class LowResourceMonitor extends AbstractLifeCycle
                     c.setAccepting(false);
                 }
             }
-            
+
-            for (EndPoint endPoint : connector.getConnectedEndPoints())
+            for ( EndPoint endPoint : connector.getConnectedEndPoints())
                 endPoint.setIdleTimeout(_lowResourcesIdleTimeout);
         }
     }
@@ -379,7 +423,7 @@ public class LowResourceMonitor extends AbstractLifeCycle
             for (EndPoint endPoint : connector.getConnectedEndPoints())
                 endPoint.setIdleTimeout(connector.getIdleTimeout());
         }
-        
+
         for (AbstractConnector connector : _acceptingConnectors)
         {
             connector.setAccepting(true);
@@ -387,15 +431,219 @@ public class LowResourceMonitor extends AbstractLifeCycle
         _acceptingConnectors.clear();
     }
 
-    private String low(String reasons, String newReason)
+    protected String low(String reasons, String newReason)
     {
         if (reasons==null)
             return newReason;
         return reasons+", "+newReason;
     }
 
-
     private static class LRMScheduler extends ScheduledExecutorScheduler
     {
     }
+
+    interface LowResourceCheck
+    {
+        boolean isLowOnResources();
+
+        String getReason();
+    }
+
+    //------------------------------------------------------
+    // default implementations for backward compat
+    //------------------------------------------------------
+
+    public class MainThreadPoolLowResourceCheck implements LowResourceCheck
+    {
+        private String reason;
+
+        public MainThreadPoolLowResourceCheck()
+        {
+            // no op
+        }
+
+        @Override
+        public boolean isLowOnResources()
+        {
+            ThreadPool serverThreads = _server.getThreadPool();
+            if (serverThreads.isLowOnThreads())
+            {
+                reason="Server low on threads: "+serverThreads;
+                return true;
+            }
+            return false;
+        }
+
+        @Override
+        public String getReason()
+        {
+            return reason;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Check if the server ThreadPool is lowOnThreads";
+        }
+    }
+
+    public class ConnectorsThreadPoolLowResourceCheck implements LowResourceCheck
+    {
+        private String reason;
+
+        public ConnectorsThreadPoolLowResourceCheck()
+        {
+            // no op
+        }
+
+        @Override
+        public boolean isLowOnResources()
+        {
+            ThreadPool serverThreads = _server.getThreadPool();
+            if(serverThreads.isLowOnThreads())
+            {
+                reason ="Server low on threads: "+serverThreads.getThreads()+", idleThreads:"+serverThreads.getIdleThreads();
+                return true;
+            }
+            for(Connector connector : getMonitoredConnectors())
+            {
+                Executor executor = connector.getExecutor();
+                if (executor instanceof ThreadPool && executor!=serverThreads)
+                {
+                    ThreadPool connectorThreads=(ThreadPool)executor;
+                    if (connectorThreads.isLowOnThreads())
+                    {
+                        reason ="Connector low on threads: "+connectorThreads;
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
+
+        @Override
+        public String getReason()
+        {
+            return reason;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Check if the ThreadPool from monitored connectors are lowOnThreads and if all connections number is higher than the allowed maxConnection";
+        }
+    }
+
+    @ManagedObject("Check max allowed connections on connectors")
+    public class MaxConnectionsLowResourceCheck implements LowResourceCheck
+    {
+        private String reason;
+        private int maxConnections;
+
+        public MaxConnectionsLowResourceCheck(int maxConnections)
+        {
+            this.maxConnections = maxConnections;
+        }
+
+        /**
+         * @return The maximum connections allowed for the monitored connectors before low resource handling is activated
+         * @deprecated Replaced by ConnectionLimit
+         */
+        @ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated")
+        @Deprecated
+        public int getMaxConnections()
+        {
+            return maxConnections;
+        }
+
+        /**
+         * @param maxConnections The maximum connections before low resources state is triggered
+         * @deprecated Replaced by ConnectionLimit
+         */
+        @Deprecated
+        public void setMaxConnections(int maxConnections)
+        {
+            if (maxConnections>0)
+                LOG.warn("LowResourceMonitor.setMaxConnections is deprecated. Use ConnectionLimit.");
+            this.maxConnections = maxConnections;
+        }
+
+        @Override
+        public boolean isLowOnResources()
+        {
+            int connections=0;
+            for(Connector connector : getMonitoredConnectors())
+            {
+                connections+=connector.getConnectedEndPoints().size();
+            }
+            if (maxConnections>0 && connections>maxConnections)
+            {
+                reason ="Max Connections exceeded: "+connections+">"+maxConnections;
+                return true;
+            }
+            return false;
+        }
+
+        @Override
+        public String getReason()
+        {
+            return reason;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "All connections number is higher than the allowed maxConnection";
+        }
+    }
+
+    public class MemoryLowResourceCheck implements LowResourceCheck
+    {
+        private String reason;
+        private long maxMemory;
+
+        public MemoryLowResourceCheck(long maxMemory)
+        {
+            this.maxMemory = maxMemory;
+        }
+
+        @Override
+        public boolean isLowOnResources()
+        {
+            long memory=Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory();
+            if (maxMemory>0 && memory>maxMemory)
+            {
+                reason = "Max memory exceeded: "+memory+">"+maxMemory;
+                return true;
+            }
+            return false;
+        }
+
+        public long getMaxMemory()
+        {
+            return maxMemory;
+        }
+
+        /**
+         * @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered.
+         */
+        public void setMaxMemory( long maxMemoryBytes )
+        {
+            this.maxMemory = maxMemoryBytes;
+        }
+
+        @Override
+        public String getReason()
+        {
+            return reason;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Check if used memory is higher than the allowed max memory";
+        }
+    }
+
+
 }

jetty-server/src/test/java/org/eclipse/jetty/server/CustomResourcesMonitorTest.java

@@ -0,0 +1,194 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.server;
+
+
+import org.eclipse.jetty.toolchain.test.AdvancedRunner;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.util.thread.TimerScheduler;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.*;
+
+@RunWith(AdvancedRunner.class)
+public class CustomResourcesMonitorTest
+{
+    Server _server;
+    ServerConnector _connector;
+    FileOnDirectoryMonitor _fileOnDirectoryMonitor;
+    Path _monitoredPath;
+    LowResourceMonitor _lowResourceMonitor;
+
+    @Before
+    public void before() throws Exception
+    {
+        _server = new Server();
+
+        _server.addBean(new TimerScheduler());
+        
+        _connector = new ServerConnector(_server);
+        _connector.setPort(0);
+        _connector.setIdleTimeout(35000);
+        _server.addConnector(_connector);
+
+        _server.setHandler(new DumpHandler());
+
+        _monitoredPath = Files.createTempDirectory( "jetty_test" );
+        _fileOnDirectoryMonitor=new FileOnDirectoryMonitor(_monitoredPath);
+        _lowResourceMonitor = new LowResourceMonitor(_server);
+        _server.addBean( _lowResourceMonitor );
+        _lowResourceMonitor.addLowResourceCheck( _fileOnDirectoryMonitor );
+        _server.start();
+    }
+    
+    @After
+    public void after() throws Exception
+    {
+        _server.stop();
+    }
+
+    @Test
+    public void testFileOnDirectoryMonitor() throws Exception
+    {
+        int monitorPeriod = _lowResourceMonitor.getPeriod();
+        int lowResourcesIdleTimeout = _lowResourceMonitor.getLowResourcesIdleTimeout();
+        assertThat(lowResourcesIdleTimeout, Matchers.lessThanOrEqualTo(monitorPeriod));
+
+        int maxLowResourcesTime = 5 * monitorPeriod;
+        _lowResourceMonitor.setMaxLowResourcesTime(maxLowResourcesTime);
+        assertFalse(_fileOnDirectoryMonitor.isLowOnResources());
+
+        try(Socket socket0 = new Socket("localhost",_connector.getLocalPort()))
+        {
+            Path tmpFile = Files.createTempFile( _monitoredPath, "yup", ".tmp" );
+            // Write a file
+            Files.write(tmpFile, "foobar".getBytes());
+
+            // Wait a couple of monitor periods so that
+            // fileOnDirectoryMonitor detects it is in low mode.
+            Thread.sleep(2 * monitorPeriod);
+            assertTrue(_fileOnDirectoryMonitor.isLowOnResources());
+
+
+            // We already waited enough for fileOnDirectoryMonitor to close socket0.
+            assertEquals(-1, socket0.getInputStream().read());
+
+            // New connections are not affected by the
+            // low mode until maxLowResourcesTime elapses.
+            try(Socket socket1 = new Socket("localhost",_connector.getLocalPort()))
+            {
+                // Set a very short read timeout so we can test if the server closed.
+                socket1.setSoTimeout(1);
+                InputStream input1 = socket1.getInputStream();
+
+                assertTrue(_fileOnDirectoryMonitor.isLowOnResources());
+                try
+                {
+                    input1.read();
+                    fail();
+                }
+                catch (SocketTimeoutException expected)
+                {
+                }
+
+                // Wait a couple of lowResources idleTimeouts.
+                Thread.sleep(2 * lowResourcesIdleTimeout);
+
+                // Verify the new socket is still open.
+                assertTrue(_fileOnDirectoryMonitor.isLowOnResources());
+                try
+                {
+                    input1.read();
+                    fail();
+                }
+                catch (SocketTimeoutException expected)
+                {
+                }
+
+                Files.delete( tmpFile );
+
+                // Let the maxLowResourcesTime elapse.
+                Thread.sleep(maxLowResourcesTime);
+                assertFalse(_fileOnDirectoryMonitor.isLowOnResources());
+            }
+        }
+    }
+
+
+    static class FileOnDirectoryMonitor implements LowResourceMonitor.LowResourceCheck
+    {
+        private static final Logger LOG = Log.getLogger( FileOnDirectoryMonitor.class);
+
+        private final Path _pathToMonitor;
+
+        private String reason;
+
+        public FileOnDirectoryMonitor(Path pathToMonitor )
+        {
+            _pathToMonitor = pathToMonitor;
+        }
+
+        @Override
+        public boolean isLowOnResources()
+        {
+            try
+            {
+                Stream<Path> paths = Files.list( _pathToMonitor );
+                List<Path> content = paths.collect( Collectors.toList() );
+                if(!content.isEmpty())
+                {
+                    reason= "directory not empty so enable low resources";
+                    return true;
+                }
+            }
+            catch ( IOException e )
+            {
+                LOG.info( "ignore issue looking at directory content", e );
+            }
+            return false;
+        }
+
+        @Override
+        public String getReason()
+        {
+            return reason;
+        }
+
+        @Override
+        public String toString()
+        {
+            return getClass().getName();
+        }
+    }
+}

jetty-server/src/test/java/org/eclipse/jetty/server/LowResourcesMonitorTest.java

@@ -19,27 +19,27 @@
 package org.eclipse.jetty.server;
 
 
-import java.io.InputStream;
-import java.net.Socket;
-import java.net.SocketTimeoutException;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.concurrent.CountDownLatch;
-
 import org.eclipse.jetty.toolchain.test.AdvancedRunner;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.eclipse.jetty.util.thread.TimerScheduler;
 import org.hamcrest.Matchers;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
+import java.io.InputStream;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.*;
 
 @RunWith(AdvancedRunner.class)
 public class LowResourcesMonitorTest
@@ -48,7 +48,7 @@ public class LowResourcesMonitorTest
     Server _server;
     ServerConnector _connector;
     LowResourceMonitor _lowResourcesMonitor;
-    
+
     @Before
     public void before() throws Exception
     {
@@ -71,6 +71,7 @@ public class LowResourcesMonitorTest
         _lowResourcesMonitor.setLowResourcesIdleTimeout(200);
         _lowResourcesMonitor.setMaxConnections(20);
         _lowResourcesMonitor.setPeriod(900);
+        _lowResourcesMonitor.setMonitoredConnectors( Collections.singleton( _connector ) );
         _server.addBean(_lowResourcesMonitor);
 
         _server.start();
@@ -86,38 +87,35 @@ public class LowResourcesMonitorTest
     @Test
     public void testLowOnThreads() throws Exception
     {
+        _lowResourcesMonitor.setMonitorThreads(true);
         Thread.sleep(1200);
         _threadPool.setMaxThreads(_threadPool.getThreads()-_threadPool.getIdleThreads()+10);
         Thread.sleep(1200);
-        Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
+        assertFalse(_lowResourcesMonitor.getReasons(), _lowResourcesMonitor.isLowOnResources());
         
         final CountDownLatch latch = new CountDownLatch(1);
         
         for (int i=0;i<100;i++)
         {
-            _threadPool.execute(new Runnable()
+            _threadPool.execute(() ->
             {
-                @Override
+                try
-                public void run()
                 {
-                    try
+                    latch.await();
-                    {
+                }
-                        latch.await();
+                catch (InterruptedException e)
-                    }
+                {
-                    catch (InterruptedException e)
+                    e.printStackTrace();
-                    {
-                        e.printStackTrace();
-                    }
                 }
             });
         }
         
         Thread.sleep(1200);
-        Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
+        assertTrue(_lowResourcesMonitor.isLowOnResources());
         
         latch.countDown();
         Thread.sleep(1200);
-        Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());      
+        assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
     }
     
 
@@ -125,10 +123,13 @@ public class LowResourcesMonitorTest
     public void testNotAccepting() throws Exception
     {
         _lowResourcesMonitor.setAcceptingInLowResources(false);
+        _lowResourcesMonitor.setMonitorThreads( true );
         Thread.sleep(1200);
-        _threadPool.setMaxThreads(_threadPool.getThreads()-_threadPool.getIdleThreads()+10);
+        int maxThreads = _threadPool.getThreads()-_threadPool.getIdleThreads()+10;
+        System.out.println("maxThreads:"+maxThreads);
+        _threadPool.setMaxThreads(maxThreads);
         Thread.sleep(1200);
-        Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
+        assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
         
         for (AbstractConnector c : _server.getBeans(AbstractConnector.class))
             assertThat(c.isAccepting(),Matchers.is(true));
@@ -136,31 +137,27 @@ public class LowResourcesMonitorTest
         final CountDownLatch latch = new CountDownLatch(1);
         for (int i=0;i<100;i++)
         {
-            _threadPool.execute(new Runnable()
+            _threadPool.execute(() ->
             {
-                @Override
+                try
-                public void run()
                 {
-                    try
+                    latch.await();
-                    {
+                }
-                        latch.await();
+                catch (InterruptedException e)
-                    }
+                {
-                    catch (InterruptedException e)
+                    e.printStackTrace();
-                    {
-                        e.printStackTrace();
-                    }
                 }
             });
         }
         
         Thread.sleep(1200);
-        Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
+        assertTrue(_lowResourcesMonitor.isLowOnResources());
         for (AbstractConnector c : _server.getBeans(AbstractConnector.class))
             assertThat(c.isAccepting(),Matchers.is(false));
         
         latch.countDown();
         Thread.sleep(1200);
-        Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());   
+        assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
         for (AbstractConnector c : _server.getBeans(AbstractConnector.class))
             assertThat(c.isAccepting(),Matchers.is(true));   
     }
@@ -172,7 +169,7 @@ public class LowResourcesMonitorTest
     {
         _lowResourcesMonitor.setMaxMemory(Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory()+(100*1024*1024));
         Thread.sleep(1200);
-        Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
+        assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
 
         byte[] data = new byte[100*1024*1024];
         Arrays.fill(data,(byte)1);
@@ -180,13 +177,13 @@ public class LowResourcesMonitorTest
         assertThat(hash,not(equalTo(0)));
 
         Thread.sleep(1200);
-        Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
+        assertTrue(_lowResourcesMonitor.isLowOnResources());
         data=null;
         System.gc();
         System.gc();
 
         Thread.sleep(1200);
-        Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());   
+        assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
     }
     
 
@@ -194,26 +191,27 @@ public class LowResourcesMonitorTest
     public void testMaxConnectionsAndMaxIdleTime() throws Exception
     {
         _lowResourcesMonitor.setMaxMemory(0);
-        Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
+        assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
 
+        assertEquals( 20, _lowResourcesMonitor.getMaxConnections() );
         Socket[] socket = new Socket[_lowResourcesMonitor.getMaxConnections()+1];
         for (int i=0;i<socket.length;i++)
             socket[i]=new Socket("localhost",_connector.getLocalPort());
         
         Thread.sleep(1200);
-        Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
+        assertTrue(_lowResourcesMonitor.isLowOnResources());
 
         try(Socket newSocket = new Socket("localhost",_connector.getLocalPort()))
         {
             // wait for low idle time to close sockets, but not new Socket
             Thread.sleep(1200);
-            Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());   
+            assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
 
             for (int i=0;i<socket.length;i++)
-                Assert.assertEquals(-1,socket[i].getInputStream().read());
+                assertEquals(-1,socket[i].getInputStream().read());
 
             newSocket.getOutputStream().write("GET / HTTP/1.0\r\n\r\n".getBytes(StandardCharsets.UTF_8));
-            Assert.assertEquals('H',newSocket.getInputStream().read());
+            assertEquals('H',newSocket.getInputStream().read());
         }
     }
     
@@ -222,11 +220,11 @@ public class LowResourcesMonitorTest
     {
         int monitorPeriod = _lowResourcesMonitor.getPeriod();
         int lowResourcesIdleTimeout = _lowResourcesMonitor.getLowResourcesIdleTimeout();
-        Assert.assertThat(lowResourcesIdleTimeout, Matchers.lessThan(monitorPeriod));
+        assertThat(lowResourcesIdleTimeout, Matchers.lessThan(monitorPeriod));
 
         int maxLowResourcesTime = 5 * monitorPeriod;
         _lowResourcesMonitor.setMaxLowResourcesTime(maxLowResourcesTime);
-        Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
+        assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
 
         try(Socket socket0 = new Socket("localhost",_connector.getLocalPort()))
         {
@@ -236,10 +234,10 @@ public class LowResourcesMonitorTest
             // Wait a couple of monitor periods so that
             // lowResourceMonitor detects it is in low mode.
             Thread.sleep(2 * monitorPeriod);
-            Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
+            assertTrue(_lowResourcesMonitor.isLowOnResources());
 
             // We already waited enough for lowResourceMonitor to close socket0.
-            Assert.assertEquals(-1, socket0.getInputStream().read());
+            assertEquals(-1, socket0.getInputStream().read());
 
             // New connections are not affected by the
             // low mode until maxLowResourcesTime elapses.
@@ -249,11 +247,11 @@ public class LowResourcesMonitorTest
                 socket1.setSoTimeout(1);
                 InputStream input1 = socket1.getInputStream();
 
-                Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
+                assertTrue(_lowResourcesMonitor.isLowOnResources());
                 try
                 {
                     input1.read();
-                    Assert.fail();
+                    fail();
                 }
                 catch (SocketTimeoutException expected)
                 {
@@ -263,22 +261,21 @@ public class LowResourcesMonitorTest
                 Thread.sleep(2 * lowResourcesIdleTimeout);
 
                 // Verify the new socket is still open.
-                Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
+                assertTrue(_lowResourcesMonitor.isLowOnResources());
                 try
                 {
                     input1.read();
-                    Assert.fail();
+                    fail();
                 }
                 catch (SocketTimeoutException expected)
                 {
                 }
-
                 // Let the maxLowResourcesTime elapse.
                 Thread.sleep(maxLowResourcesTime);
 
-                Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
+                assertTrue(_lowResourcesMonitor.isLowOnResources());
                 // Now also the new socket should be closed.
-                Assert.assertEquals(-1, input1.read());
+                assertEquals(-1, input1.read());
             }
         }
     }