From 44e57f21701b7fda71542b8af0b448754a84667c Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Thu, 23 Aug 2018 21:42:37 +1000 Subject: [PATCH] Issue #2775 LowResourceMonitor extendable (#2812) * make LowResourceMonitor extendable #2775 Signed-off-by: olivier lamy --- .../main/config/etc/jetty-lowresources.xml | 4 +- .../jetty/server/LowResourceMonitor.java | 606 ++++++++++++------ .../server/CustomResourcesMonitorTest.java | 194 ++++++ .../jetty/server/LowResourcesMonitorTest.java | 113 ++-- 4 files changed, 678 insertions(+), 239 deletions(-) create mode 100644 jetty-server/src/test/java/org/eclipse/jetty/server/CustomResourcesMonitorTest.java diff --git a/jetty-server/src/main/config/etc/jetty-lowresources.xml b/jetty-server/src/main/config/etc/jetty-lowresources.xml index 273ccd3ce6e..4e9fbd8d1ea 100644 --- a/jetty-server/src/main/config/etc/jetty-lowresources.xml +++ b/jetty-server/src/main/config/etc/jetty-lowresources.xml @@ -8,7 +8,7 @@ - + @@ -16,7 +16,7 @@ - + diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/LowResourceMonitor.java b/jetty-server/src/main/java/org/eclipse/jetty/server/LowResourceMonitor.java index e172b9fbeb7..742958ddba7 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/LowResourceMonitor.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/LowResourceMonitor.java @@ -18,6 +18,17 @@ package org.eclipse.jetty.server; +import org.eclipse.jetty.io.EndPoint; +import org.eclipse.jetty.util.annotation.ManagedAttribute; +import org.eclipse.jetty.util.annotation.ManagedObject; +import org.eclipse.jetty.util.annotation.Name; +import org.eclipse.jetty.util.component.ContainerLifeCycle; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.log.Logger; +import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler; +import org.eclipse.jetty.util.thread.Scheduler; +import org.eclipse.jetty.util.thread.ThreadPool; + import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -27,66 +38,46 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import org.eclipse.jetty.io.EndPoint; -import org.eclipse.jetty.util.annotation.ManagedAttribute; -import org.eclipse.jetty.util.annotation.ManagedObject; -import org.eclipse.jetty.util.annotation.Name; -import org.eclipse.jetty.util.component.AbstractLifeCycle; -import org.eclipse.jetty.util.log.Log; -import org.eclipse.jetty.util.log.Logger; -import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler; -import org.eclipse.jetty.util.thread.Scheduler; -import org.eclipse.jetty.util.thread.ThreadPool; - /** - * A monitor for low resources - *

- * An instance of this class will monitor all the connectors of a server (or a set of connectors - * configured with {@link #setMonitoredConnectors(Collection)}) for a low resources state. - *

- * Low resources can be detected by: + * + * A monitor for low resources, low resources can be detected by: *

    *
  • {@link ThreadPool#isLowOnThreads()} if {@link Connector#getExecutor()} is * an instance of {@link ThreadPool} and {@link #setMonitorThreads(boolean)} is true.
    • *
  • If {@link #setMaxMemory(long)} is non zero then low resources is detected if the JVMs * {@link Runtime} instance has {@link Runtime#totalMemory()} minus {@link Runtime#freeMemory()} * greater than {@link #getMaxMemory()}
    • - *
  • If {@link #setMaxConnections(int)} is non zero then low resources is dected if the total number + *
  • If {@link #setMaxConnections(int)} is non zero then low resources is detected if the total number * of connections exceeds {@link #getMaxConnections()}. This feature is deprecated and replaced by * {@link ConnectionLimit}
    • *
- *

- * Once low resources state is detected, the cause is logged and all existing connections returned - * by {@link Connector#getConnectedEndPoints()} have {@link EndPoint#setIdleTimeout(long)} set - * to {@link #getLowResourcesIdleTimeout()}. New connections are not affected, however if the low - * resources state persists for more than {@link #getMaxLowResourcesTime()}, then the - * {@link #getLowResourcesIdleTimeout()} to all connections again. Once the low resources state is - * cleared, the idle timeout is reset to the connector default given by {@link Connector#getIdleTimeout()}. - *

- * If {@link #setAcceptingInLowResources(boolean)} is set to false (Default is true), then no new connections - * are accepted when in low resources state. + * */ @ManagedObject ("Monitor for low resource conditions and activate a low resource mode if detected") -public class LowResourceMonitor extends AbstractLifeCycle +public class LowResourceMonitor extends ContainerLifeCycle { private static final Logger LOG = Log.getLogger(LowResourceMonitor.class); - private final Server _server; + + protected final Server _server; private Scheduler _scheduler; private Connector[] _monitoredConnectors; private Set _acceptingConnectors = new HashSet<>(); private int _period=1000; - private int _maxConnections; - private long _maxMemory; + + private int _lowResourcesIdleTimeout=1000; private int _maxLowResourcesTime=0; - private boolean _monitorThreads=true; + private final AtomicBoolean _low = new AtomicBoolean(); - private String _cause; + private String _reasons; + private long _lowStarted; private boolean _acceptingInLowResources = true; + private Set _lowResourceChecks = new HashSet<>(); + private final Runnable _monitor = new Runnable() { @Override @@ -95,14 +86,86 @@ public class LowResourceMonitor extends AbstractLifeCycle if (isRunning()) { monitor(); - _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS); + _scheduler.schedule( _monitor, _period, TimeUnit.MILLISECONDS); } } }; public LowResourceMonitor(@Name("server") Server server) { - _server=server; + _server = server; + } + + @ManagedAttribute("True if low available threads status is monitored") + public boolean getMonitorThreads() + { + return !getBeans(ConnectorsThreadPoolLowResourceCheck.class).isEmpty(); + } + + /** + * @param monitorThreads If true, check connectors executors to see if they are + * {@link ThreadPool} instances that are low on threads. + */ + public void setMonitorThreads(boolean monitorThreads) + { + if(monitorThreads) + // already configured? + if ( !getMonitorThreads() ) addLowResourceCheck( new ConnectorsThreadPoolLowResourceCheck() ); + else + getBeans(ConnectorsThreadPoolLowResourceCheck.class).forEach(this::removeBean); + } + + /** + * @return The maximum connections allowed for the monitored connectors before low resource handling is activated + * @deprecated Replaced by ConnectionLimit + */ + @ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated") + @Deprecated + public int getMaxConnections() + { + for(MaxConnectionsLowResourceCheck lowResourceCheck : getBeans(MaxConnectionsLowResourceCheck.class)) + { + if (lowResourceCheck.getMaxConnections()>0) + { + return lowResourceCheck.getMaxConnections(); + } + } + return -1; + } + + /** + * @param maxConnections The maximum connections before low resources state is triggered + * @deprecated Replaced by ConnectionLimit + */ + @Deprecated + public void setMaxConnections(int maxConnections) + { + if (maxConnections>0) + { + if (getBeans(MaxConnectionsLowResourceCheck.class).isEmpty()) + { + addLowResourceCheck(new MaxConnectionsLowResourceCheck(maxConnections)); + } else + { + getBeans(MaxConnectionsLowResourceCheck.class).forEach( c -> c.setMaxConnections( maxConnections ) ); + } + } + else + { + getBeans(ConnectorsThreadPoolLowResourceCheck.class).forEach(this::removeBean); + } + } + + + @ManagedAttribute("The reasons the monitored connectors are low on resources") + public String getReasons() + { + return _reasons; + } + + protected void setReasons(String reasons) + { + _reasons = reasons; } @ManagedAttribute("Are the monitored connectors low on resources?") @@ -111,24 +174,39 @@ public class LowResourceMonitor extends AbstractLifeCycle return _low.get(); } + protected boolean enableLowOnResources(boolean expectedValue, boolean newValue) + { + return _low.compareAndSet(expectedValue, newValue); + } + @ManagedAttribute("The reason(s) the monitored connectors are low on resources") public String getLowResourcesReasons() { return _reasons; } + protected void setLowResourcesReasons(String reasons) + { + _reasons = reasons; + } + @ManagedAttribute("Get the timestamp in ms since epoch that low resources state started") public long getLowResourcesStarted() { return _lowStarted; } + public void setLowResourcesStarted(long lowStarted) + { + _lowStarted = lowStarted; + } + @ManagedAttribute("The monitored connectors. If null then all server connectors are monitored") public Collection getMonitoredConnectors() { if (_monitoredConnectors==null) return Collections.emptyList(); - return Arrays.asList(_monitoredConnectors); + return Arrays.asList( _monitoredConnectors); } /** @@ -142,6 +220,13 @@ public class LowResourceMonitor extends AbstractLifeCycle _monitoredConnectors = monitoredConnectors.toArray(new Connector[monitoredConnectors.size()]); } + protected Connector[] getMonitoredOrServerConnectors() + { + if (_monitoredConnectors!=null && _monitoredConnectors.length>0) + return _monitoredConnectors; + return _server.getConnectors(); + } + @ManagedAttribute("If false, new connections are not accepted while in low resources") public boolean isAcceptingInLowResources() { @@ -152,7 +237,7 @@ public class LowResourceMonitor extends AbstractLifeCycle { _acceptingInLowResources = acceptingInLowResources; } - + @ManagedAttribute("The monitor period in ms") public int getPeriod() { @@ -167,58 +252,6 @@ public class LowResourceMonitor extends AbstractLifeCycle _period = periodMS; } - @ManagedAttribute("True if low available threads status is monitored") - public boolean getMonitorThreads() - { - return _monitorThreads; - } - - /** - * @param monitorThreads If true, check connectors executors to see if they are - * {@link ThreadPool} instances that are low on threads. - */ - public void setMonitorThreads(boolean monitorThreads) - { - _monitorThreads = monitorThreads; - } - - /** - * @return The maximum connections allowed for the monitored connectors before low resource handling is activated - * @deprecated Replaced by ConnectionLimit - */ - @ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated") - @Deprecated - public int getMaxConnections() - { - return _maxConnections; - } - - /** - * @param maxConnections The maximum connections before low resources state is triggered - * @deprecated Replaced by ConnectionLimit - */ - @Deprecated - public void setMaxConnections(int maxConnections) - { - if (maxConnections>0) - LOG.warn("LowResourceMonitor.setMaxConnections is deprecated. Use ConnectionLimit."); - _maxConnections = maxConnections; - } - - @ManagedAttribute("The maximum memory (in bytes) that can be used before low resources is triggered. Memory used is calculated as (totalMemory-freeMemory).") - public long getMaxMemory() - { - return _maxMemory; - } - - /** - * @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered. - */ - public void setMaxMemory(long maxMemoryBytes) - { - _maxMemory = maxMemoryBytes; - } - @ManagedAttribute("The idletimeout in ms to apply to all existing connections when low resources is detected") public int getLowResourcesIdleTimeout() { @@ -247,6 +280,99 @@ public class LowResourceMonitor extends AbstractLifeCycle _maxLowResourcesTime = maxLowResourcesTimeMS; } + @ManagedAttribute("The maximum memory (in bytes) that can be used before low resources is triggered. Memory used is calculated as (totalMemory-freeMemory).") + public long getMaxMemory() + { + Collection beans = getBeans(MemoryLowResourceCheck.class); + if(beans.isEmpty()) + { + return 0; + } + return beans.stream().findFirst().get().getMaxMemory(); + } + + /** + * @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered. + */ + public void setMaxMemory(long maxMemoryBytes) + { + if(maxMemoryBytes<=0) + { + return; + } + Collection beans = getBeans(MemoryLowResourceCheck.class); + if(beans.isEmpty()) + addLowResourceCheck( new MemoryLowResourceCheck( maxMemoryBytes ) ); + else + beans.forEach( lowResourceCheck -> lowResourceCheck.setMaxMemory( maxMemoryBytes ) ); + } + + public Set getLowResourceChecks() + { + return _lowResourceChecks; + } + + public void setLowResourceChecks( Set lowResourceChecks ) + { + updateBeans(_lowResourceChecks.toArray(),lowResourceChecks.toArray()); + this._lowResourceChecks = lowResourceChecks; + } + + public void addLowResourceCheck( LowResourceCheck lowResourceCheck ) + { + addBean(lowResourceCheck); + this._lowResourceChecks.add(lowResourceCheck); + } + + protected void monitor() + { + + String reasons=null; + + + for(LowResourceCheck lowResourceCheck : _lowResourceChecks) + { + if(lowResourceCheck.isLowOnResources()) + { + reasons = lowResourceCheck.toString(); + break; + } + } + + if (reasons!=null) + { + // Log the reasons if there is any change in the cause + if (!reasons.equals(getReasons())) + { + LOG.warn("Low Resources: {}",reasons); + setReasons(reasons); + } + + // Enter low resources state? + if (enableLowOnResources(false,true)) + { + setLowResourcesReasons(reasons); + setLowResourcesStarted(System.currentTimeMillis()); + setLowResources(); + } + + // Too long in low resources state? + if ( getMaxLowResourcesTime()>0 && (System.currentTimeMillis()-getLowResourcesStarted())>getMaxLowResourcesTime()) + setLowResources(); + } + else + { + if (enableLowOnResources(true,false)) + { + LOG.info("Low Resources cleared"); + setLowResourcesReasons(null); + setLowResourcesStarted(0); + setReasons(null); + clearLowResources(); + } + } + } + @Override protected void doStart() throws Exception { @@ -257,6 +383,7 @@ public class LowResourceMonitor extends AbstractLifeCycle _scheduler=new LRMScheduler(); _scheduler.start(); } + super.doStart(); _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS); @@ -265,94 +392,11 @@ public class LowResourceMonitor extends AbstractLifeCycle @Override protected void doStop() throws Exception { - if (_scheduler instanceof LRMScheduler) + if (_scheduler instanceof LRMScheduler ) _scheduler.stop(); super.doStop(); } - protected Connector[] getMonitoredOrServerConnectors() - { - if (_monitoredConnectors!=null && _monitoredConnectors.length>0) - return _monitoredConnectors; - return _server.getConnectors(); - } - - protected void monitor() - { - String reasons=null; - String cause=""; - int connections=0; - - ThreadPool serverThreads = _server.getThreadPool(); - if (_monitorThreads && serverThreads.isLowOnThreads()) - { - reasons=low(reasons,"Server low on threads: "+serverThreads); - cause+="S"; - } - - for(Connector connector : getMonitoredOrServerConnectors()) - { - connections+=connector.getConnectedEndPoints().size(); - - Executor executor = connector.getExecutor(); - if (executor instanceof ThreadPool && executor!=serverThreads) - { - ThreadPool connectorThreads=(ThreadPool)executor; - if (_monitorThreads && connectorThreads.isLowOnThreads()) - { - reasons=low(reasons,"Connector low on threads: "+connectorThreads); - cause+="T"; - } - } - } - - if (_maxConnections>0 && connections>_maxConnections) - { - reasons=low(reasons,"Max Connections exceeded: "+connections+">"+_maxConnections); - cause+="C"; - } - - long memory=Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory(); - if (_maxMemory>0 && memory>_maxMemory) - { - reasons=low(reasons,"Max memory exceeded: "+memory+">"+_maxMemory); - cause+="M"; - } - - if (reasons!=null) - { - // Log the reasons if there is any change in the cause - if (!cause.equals(_cause)) - { - LOG.warn("Low Resources: {}",reasons); - _cause=cause; - } - - // Enter low resources state? - if (_low.compareAndSet(false,true)) - { - _reasons=reasons; - _lowStarted=System.currentTimeMillis(); - setLowResources(); - } - - // Too long in low resources state? - if (_maxLowResourcesTime>0 && (System.currentTimeMillis()-_lowStarted)>_maxLowResourcesTime) - setLowResources(); - } - else - { - if (_low.compareAndSet(true,false)) - { - LOG.info("Low Resources cleared"); - _reasons=null; - _lowStarted=0; - _cause=null; - clearLowResources(); - } - } - } - protected void setLowResources() { for(Connector connector : getMonitoredOrServerConnectors()) @@ -366,8 +410,8 @@ public class LowResourceMonitor extends AbstractLifeCycle c.setAccepting(false); } } - - for (EndPoint endPoint : connector.getConnectedEndPoints()) + + for ( EndPoint endPoint : connector.getConnectedEndPoints()) endPoint.setIdleTimeout(_lowResourcesIdleTimeout); } } @@ -379,7 +423,7 @@ public class LowResourceMonitor extends AbstractLifeCycle for (EndPoint endPoint : connector.getConnectedEndPoints()) endPoint.setIdleTimeout(connector.getIdleTimeout()); } - + for (AbstractConnector connector : _acceptingConnectors) { connector.setAccepting(true); @@ -387,15 +431,219 @@ public class LowResourceMonitor extends AbstractLifeCycle _acceptingConnectors.clear(); } - private String low(String reasons, String newReason) + protected String low(String reasons, String newReason) { if (reasons==null) return newReason; return reasons+", "+newReason; } - private static class LRMScheduler extends ScheduledExecutorScheduler { } + + interface LowResourceCheck + { + boolean isLowOnResources(); + + String getReason(); + } + + //------------------------------------------------------ + // default implementations for backward compat + //------------------------------------------------------ + + public class MainThreadPoolLowResourceCheck implements LowResourceCheck + { + private String reason; + + public MainThreadPoolLowResourceCheck() + { + // no op + } + + @Override + public boolean isLowOnResources() + { + ThreadPool serverThreads = _server.getThreadPool(); + if (serverThreads.isLowOnThreads()) + { + reason="Server low on threads: "+serverThreads; + return true; + } + return false; + } + + @Override + public String getReason() + { + return reason; + } + + @Override + public String toString() + { + return "Check if the server ThreadPool is lowOnThreads"; + } + } + + public class ConnectorsThreadPoolLowResourceCheck implements LowResourceCheck + { + private String reason; + + public ConnectorsThreadPoolLowResourceCheck() + { + // no op + } + + @Override + public boolean isLowOnResources() + { + ThreadPool serverThreads = _server.getThreadPool(); + if(serverThreads.isLowOnThreads()) + { + reason ="Server low on threads: "+serverThreads.getThreads()+", idleThreads:"+serverThreads.getIdleThreads(); + return true; + } + for(Connector connector : getMonitoredConnectors()) + { + Executor executor = connector.getExecutor(); + if (executor instanceof ThreadPool && executor!=serverThreads) + { + ThreadPool connectorThreads=(ThreadPool)executor; + if (connectorThreads.isLowOnThreads()) + { + reason ="Connector low on threads: "+connectorThreads; + return true; + } + } + } + return false; + } + + @Override + public String getReason() + { + return reason; + } + + @Override + public String toString() + { + return "Check if the ThreadPool from monitored connectors are lowOnThreads and if all connections number is higher than the allowed maxConnection"; + } + } + + @ManagedObject("Check max allowed connections on connectors") + public class MaxConnectionsLowResourceCheck implements LowResourceCheck + { + private String reason; + private int maxConnections; + + public MaxConnectionsLowResourceCheck(int maxConnections) + { + this.maxConnections = maxConnections; + } + + /** + * @return The maximum connections allowed for the monitored connectors before low resource handling is activated + * @deprecated Replaced by ConnectionLimit + */ + @ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated") + @Deprecated + public int getMaxConnections() + { + return maxConnections; + } + + /** + * @param maxConnections The maximum connections before low resources state is triggered + * @deprecated Replaced by ConnectionLimit + */ + @Deprecated + public void setMaxConnections(int maxConnections) + { + if (maxConnections>0) + LOG.warn("LowResourceMonitor.setMaxConnections is deprecated. Use ConnectionLimit."); + this.maxConnections = maxConnections; + } + + @Override + public boolean isLowOnResources() + { + int connections=0; + for(Connector connector : getMonitoredConnectors()) + { + connections+=connector.getConnectedEndPoints().size(); + } + if (maxConnections>0 && connections>maxConnections) + { + reason ="Max Connections exceeded: "+connections+">"+maxConnections; + return true; + } + return false; + } + + @Override + public String getReason() + { + return reason; + } + + @Override + public String toString() + { + return "All connections number is higher than the allowed maxConnection"; + } + } + + public class MemoryLowResourceCheck implements LowResourceCheck + { + private String reason; + private long maxMemory; + + public MemoryLowResourceCheck(long maxMemory) + { + this.maxMemory = maxMemory; + } + + @Override + public boolean isLowOnResources() + { + long memory=Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory(); + if (maxMemory>0 && memory>maxMemory) + { + reason = "Max memory exceeded: "+memory+">"+maxMemory; + return true; + } + return false; + } + + public long getMaxMemory() + { + return maxMemory; + } + + /** + * @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered. + */ + public void setMaxMemory( long maxMemoryBytes ) + { + this.maxMemory = maxMemoryBytes; + } + + @Override + public String getReason() + { + return reason; + } + + @Override + public String toString() + { + return "Check if used memory is higher than the allowed max memory"; + } + } + + } diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/CustomResourcesMonitorTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/CustomResourcesMonitorTest.java new file mode 100644 index 00000000000..486cb80afe5 --- /dev/null +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/CustomResourcesMonitorTest.java @@ -0,0 +1,194 @@ +// +// ======================================================================== +// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server; + + +import org.eclipse.jetty.toolchain.test.AdvancedRunner; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.log.Logger; +import org.eclipse.jetty.util.thread.TimerScheduler; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.io.InputStream; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.*; + +@RunWith(AdvancedRunner.class) +public class CustomResourcesMonitorTest +{ + Server _server; + ServerConnector _connector; + FileOnDirectoryMonitor _fileOnDirectoryMonitor; + Path _monitoredPath; + LowResourceMonitor _lowResourceMonitor; + + @Before + public void before() throws Exception + { + _server = new Server(); + + _server.addBean(new TimerScheduler()); + + _connector = new ServerConnector(_server); + _connector.setPort(0); + _connector.setIdleTimeout(35000); + _server.addConnector(_connector); + + _server.setHandler(new DumpHandler()); + + _monitoredPath = Files.createTempDirectory( "jetty_test" ); + _fileOnDirectoryMonitor=new FileOnDirectoryMonitor(_monitoredPath); + _lowResourceMonitor = new LowResourceMonitor(_server); + _server.addBean( _lowResourceMonitor ); + _lowResourceMonitor.addLowResourceCheck( _fileOnDirectoryMonitor ); + _server.start(); + } + + @After + public void after() throws Exception + { + _server.stop(); + } + + @Test + public void testFileOnDirectoryMonitor() throws Exception + { + int monitorPeriod = _lowResourceMonitor.getPeriod(); + int lowResourcesIdleTimeout = _lowResourceMonitor.getLowResourcesIdleTimeout(); + assertThat(lowResourcesIdleTimeout, Matchers.lessThanOrEqualTo(monitorPeriod)); + + int maxLowResourcesTime = 5 * monitorPeriod; + _lowResourceMonitor.setMaxLowResourcesTime(maxLowResourcesTime); + assertFalse(_fileOnDirectoryMonitor.isLowOnResources()); + + try(Socket socket0 = new Socket("localhost",_connector.getLocalPort())) + { + Path tmpFile = Files.createTempFile( _monitoredPath, "yup", ".tmp" ); + // Write a file + Files.write(tmpFile, "foobar".getBytes()); + + // Wait a couple of monitor periods so that + // fileOnDirectoryMonitor detects it is in low mode. + Thread.sleep(2 * monitorPeriod); + assertTrue(_fileOnDirectoryMonitor.isLowOnResources()); + + + // We already waited enough for fileOnDirectoryMonitor to close socket0. + assertEquals(-1, socket0.getInputStream().read()); + + // New connections are not affected by the + // low mode until maxLowResourcesTime elapses. + try(Socket socket1 = new Socket("localhost",_connector.getLocalPort())) + { + // Set a very short read timeout so we can test if the server closed. + socket1.setSoTimeout(1); + InputStream input1 = socket1.getInputStream(); + + assertTrue(_fileOnDirectoryMonitor.isLowOnResources()); + try + { + input1.read(); + fail(); + } + catch (SocketTimeoutException expected) + { + } + + // Wait a couple of lowResources idleTimeouts. + Thread.sleep(2 * lowResourcesIdleTimeout); + + // Verify the new socket is still open. + assertTrue(_fileOnDirectoryMonitor.isLowOnResources()); + try + { + input1.read(); + fail(); + } + catch (SocketTimeoutException expected) + { + } + + Files.delete( tmpFile ); + + // Let the maxLowResourcesTime elapse. + Thread.sleep(maxLowResourcesTime); + assertFalse(_fileOnDirectoryMonitor.isLowOnResources()); + } + } + } + + + static class FileOnDirectoryMonitor implements LowResourceMonitor.LowResourceCheck + { + private static final Logger LOG = Log.getLogger( FileOnDirectoryMonitor.class); + + private final Path _pathToMonitor; + + private String reason; + + public FileOnDirectoryMonitor(Path pathToMonitor ) + { + _pathToMonitor = pathToMonitor; + } + + @Override + public boolean isLowOnResources() + { + try + { + Stream paths = Files.list( _pathToMonitor ); + List content = paths.collect( Collectors.toList() ); + if(!content.isEmpty()) + { + reason= "directory not empty so enable low resources"; + return true; + } + } + catch ( IOException e ) + { + LOG.info( "ignore issue looking at directory content", e ); + } + return false; + } + + @Override + public String getReason() + { + return reason; + } + + @Override + public String toString() + { + return getClass().getName(); + } + } +} diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/LowResourcesMonitorTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/LowResourcesMonitorTest.java index c95821a2487..7b70fbf1dab 100644 --- a/jetty-server/src/test/java/org/eclipse/jetty/server/LowResourcesMonitorTest.java +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/LowResourcesMonitorTest.java @@ -19,27 +19,27 @@ package org.eclipse.jetty.server; -import java.io.InputStream; -import java.net.Socket; -import java.net.SocketTimeoutException; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.concurrent.CountDownLatch; - import org.eclipse.jetty.toolchain.test.AdvancedRunner; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.eclipse.jetty.util.thread.TimerScheduler; import org.hamcrest.Matchers; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; +import java.io.InputStream; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; + import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.*; @RunWith(AdvancedRunner.class) public class LowResourcesMonitorTest @@ -48,7 +48,7 @@ public class LowResourcesMonitorTest Server _server; ServerConnector _connector; LowResourceMonitor _lowResourcesMonitor; - + @Before public void before() throws Exception { @@ -71,6 +71,7 @@ public class LowResourcesMonitorTest _lowResourcesMonitor.setLowResourcesIdleTimeout(200); _lowResourcesMonitor.setMaxConnections(20); _lowResourcesMonitor.setPeriod(900); + _lowResourcesMonitor.setMonitoredConnectors( Collections.singleton( _connector ) ); _server.addBean(_lowResourcesMonitor); _server.start(); @@ -86,38 +87,35 @@ public class LowResourcesMonitorTest @Test public void testLowOnThreads() throws Exception { + _lowResourcesMonitor.setMonitorThreads(true); Thread.sleep(1200); _threadPool.setMaxThreads(_threadPool.getThreads()-_threadPool.getIdleThreads()+10); Thread.sleep(1200); - Assert.assertFalse(_lowResourcesMonitor.isLowOnResources()); + assertFalse(_lowResourcesMonitor.getReasons(), _lowResourcesMonitor.isLowOnResources()); final CountDownLatch latch = new CountDownLatch(1); for (int i=0;i<100;i++) { - _threadPool.execute(new Runnable() + _threadPool.execute(() -> { - @Override - public void run() + try { - try - { - latch.await(); - } - catch (InterruptedException e) - { - e.printStackTrace(); - } + latch.await(); + } + catch (InterruptedException e) + { + e.printStackTrace(); } }); } Thread.sleep(1200); - Assert.assertTrue(_lowResourcesMonitor.isLowOnResources()); + assertTrue(_lowResourcesMonitor.isLowOnResources()); latch.countDown(); Thread.sleep(1200); - Assert.assertFalse(_lowResourcesMonitor.isLowOnResources()); + assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources()); } @@ -125,10 +123,13 @@ public class LowResourcesMonitorTest public void testNotAccepting() throws Exception { _lowResourcesMonitor.setAcceptingInLowResources(false); + _lowResourcesMonitor.setMonitorThreads( true ); Thread.sleep(1200); - _threadPool.setMaxThreads(_threadPool.getThreads()-_threadPool.getIdleThreads()+10); + int maxThreads = _threadPool.getThreads()-_threadPool.getIdleThreads()+10; + System.out.println("maxThreads:"+maxThreads); + _threadPool.setMaxThreads(maxThreads); Thread.sleep(1200); - Assert.assertFalse(_lowResourcesMonitor.isLowOnResources()); + assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources()); for (AbstractConnector c : _server.getBeans(AbstractConnector.class)) assertThat(c.isAccepting(),Matchers.is(true)); @@ -136,31 +137,27 @@ public class LowResourcesMonitorTest final CountDownLatch latch = new CountDownLatch(1); for (int i=0;i<100;i++) { - _threadPool.execute(new Runnable() + _threadPool.execute(() -> { - @Override - public void run() + try { - try - { - latch.await(); - } - catch (InterruptedException e) - { - e.printStackTrace(); - } + latch.await(); + } + catch (InterruptedException e) + { + e.printStackTrace(); } }); } Thread.sleep(1200); - Assert.assertTrue(_lowResourcesMonitor.isLowOnResources()); + assertTrue(_lowResourcesMonitor.isLowOnResources()); for (AbstractConnector c : _server.getBeans(AbstractConnector.class)) assertThat(c.isAccepting(),Matchers.is(false)); latch.countDown(); Thread.sleep(1200); - Assert.assertFalse(_lowResourcesMonitor.isLowOnResources()); + assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources()); for (AbstractConnector c : _server.getBeans(AbstractConnector.class)) assertThat(c.isAccepting(),Matchers.is(true)); } @@ -172,7 +169,7 @@ public class LowResourcesMonitorTest { _lowResourcesMonitor.setMaxMemory(Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory()+(100*1024*1024)); Thread.sleep(1200); - Assert.assertFalse(_lowResourcesMonitor.isLowOnResources()); + assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources()); byte[] data = new byte[100*1024*1024]; Arrays.fill(data,(byte)1); @@ -180,13 +177,13 @@ public class LowResourcesMonitorTest assertThat(hash,not(equalTo(0))); Thread.sleep(1200); - Assert.assertTrue(_lowResourcesMonitor.isLowOnResources()); + assertTrue(_lowResourcesMonitor.isLowOnResources()); data=null; System.gc(); System.gc(); Thread.sleep(1200); - Assert.assertFalse(_lowResourcesMonitor.isLowOnResources()); + assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources()); } @@ -194,26 +191,27 @@ public class LowResourcesMonitorTest public void testMaxConnectionsAndMaxIdleTime() throws Exception { _lowResourcesMonitor.setMaxMemory(0); - Assert.assertFalse(_lowResourcesMonitor.isLowOnResources()); + assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources()); + assertEquals( 20, _lowResourcesMonitor.getMaxConnections() ); Socket[] socket = new Socket[_lowResourcesMonitor.getMaxConnections()+1]; for (int i=0;i