Eclipse
/
jetty.project
mirror of https://github.com/jetty/jetty.project.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue #2775 LowResourceMonitor extendable (#2812) 

* make LowResourceMonitor extendable #2775

Signed-off-by: olivier lamy <oliver.lamy@gmail.com>
This commit is contained in:
Olivier Lamy 2018-08-23 21:42:37 +10:00 committed by GitHub
parent 460f3fcb9b
commit 44e57f2170
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 678 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
jetty-server/src/main/config/etc/jetty-lowresources.xml
View File

@ -8,7 +8,7 @@
<Configure id="Server" class="org.eclipse.jetty.server.Server">
<Call name="addBean">
<Arg>
<New class="org.eclipse.jetty.server.LowResourceMonitor">
<New id="lowResourceMonitor" class="org.eclipse.jetty.server.LowResourceMonitor">
<Arg name="server"><Ref refid='Server'/></Arg>
<Set name="period"><Property name="jetty.lowresources.period" deprecated="lowresources.period" default="1000"/></Set>
<Set name="lowResourcesIdleTimeout"><Property name="jetty.lowresources.idleTimeout" deprecated="lowresources.lowResourcesIdleTimeout" default="1000"/></Set>
@ -16,7 +16,7 @@
<Set name="maxConnections"><Property name="jetty.lowresources.maxConnections" deprecated="lowresources.maxConnections" default="0"/></Set>
<Set name="maxMemory"><Property name="jetty.lowresources.maxMemory" deprecated="lowresources.maxMemory" default="0"/></Set>
<Set name="maxLowResourcesTime"><Property name="jetty.lowresources.maxLowResourcesTime" deprecated="lowresources.maxLowResourcesTime" default="5000"/></Set>
<Set name="acceptingInLowResources"><Property name="jetty.lowresources.accepting" default="true"/></Set>
<Set name="acceptingInLowResources"><Property name="jetty.lowresources.accepting" default="true"/></Set>
</New>
</Arg>
</Call>

606
jetty-server/src/main/java/org/eclipse/jetty/server/LowResourceMonitor.java
View File

@ -18,6 +18,17 @@
package org.eclipse.jetty.server;
import org.eclipse.jetty.io.EndPoint;
import org.eclipse.jetty.util.annotation.ManagedAttribute;
import org.eclipse.jetty.util.annotation.ManagedObject;
import org.eclipse.jetty.util.annotation.Name;
import org.eclipse.jetty.util.component.ContainerLifeCycle;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.log.Logger;
import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
import org.eclipse.jetty.util.thread.Scheduler;
import org.eclipse.jetty.util.thread.ThreadPool;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@ -27,66 +38,46 @@ import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.eclipse.jetty.io.EndPoint;
import org.eclipse.jetty.util.annotation.ManagedAttribute;
import org.eclipse.jetty.util.annotation.ManagedObject;
import org.eclipse.jetty.util.annotation.Name;
import org.eclipse.jetty.util.component.AbstractLifeCycle;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.log.Logger;
import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
import org.eclipse.jetty.util.thread.Scheduler;
import org.eclipse.jetty.util.thread.ThreadPool;
/**
* A monitor for low resources
* <p>
* An instance of this class will monitor all the connectors of a server (or a set of connectors
* configured with {@link #setMonitoredConnectors(Collection)}) for a low resources state.
* <p>
* Low resources can be detected by:
*
* A monitor for low resources, low resources can be detected by:
* <ul>
* <li>{@link ThreadPool#isLowOnThreads()} if {@link Connector#getExecutor()} is
* an instance of {@link ThreadPool} and {@link #setMonitorThreads(boolean)} is true.</li>
* <li>If {@link #setMaxMemory(long)} is non zero then low resources is detected if the JVMs
* {@link Runtime} instance has {@link Runtime#totalMemory()} minus {@link Runtime#freeMemory()}
* greater than {@link #getMaxMemory()}</li>
* <li>If {@link #setMaxConnections(int)} is non zero then low resources is dected if the total number
* <li>If {@link #setMaxConnections(int)} is non zero then low resources is detected if the total number
* of connections exceeds {@link #getMaxConnections()}. This feature is deprecated and replaced by
* {@link ConnectionLimit}</li>
* </ul>
* <p>
* Once low resources state is detected, the cause is logged and all existing connections returned
* by {@link Connector#getConnectedEndPoints()} have {@link EndPoint#setIdleTimeout(long)} set
* to {@link #getLowResourcesIdleTimeout()}. New connections are not affected, however if the low
* resources state persists for more than {@link #getMaxLowResourcesTime()}, then the
* {@link #getLowResourcesIdleTimeout()} to all connections again. Once the low resources state is
* cleared, the idle timeout is reset to the connector default given by {@link Connector#getIdleTimeout()}.
* <p>
* If {@link #setAcceptingInLowResources(boolean)} is set to false (Default is true), then no new connections
* are accepted when in low resources state.
*
*/
@ManagedObject ("Monitor for low resource conditions and activate a low resource mode if detected")
public class LowResourceMonitor extends AbstractLifeCycle
public class LowResourceMonitor extends ContainerLifeCycle
{
private static final Logger LOG = Log.getLogger(LowResourceMonitor.class);
private final Server _server;
protected final Server _server;
private Scheduler _scheduler;
private Connector[] _monitoredConnectors;
private Set<AbstractConnector> _acceptingConnectors = new HashSet<>();
private int _period=1000;
private int _maxConnections;
private long _maxMemory;
private int _lowResourcesIdleTimeout=1000;
private int _maxLowResourcesTime=0;
private boolean _monitorThreads=true;
private final AtomicBoolean _low = new AtomicBoolean();
private String _cause;
private String _reasons;
private long _lowStarted;
private boolean _acceptingInLowResources = true;
private Set<LowResourceCheck> _lowResourceChecks = new HashSet<>();
private final Runnable _monitor = new Runnable()
{
@Override
@ -95,14 +86,86 @@ public class LowResourceMonitor extends AbstractLifeCycle
if (isRunning())
{
monitor();
_scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
_scheduler.schedule( _monitor, _period, TimeUnit.MILLISECONDS);
}
}
};
public LowResourceMonitor(@Name("server") Server server)
{
_server=server;
_server = server;
}
@ManagedAttribute("True if low available threads status is monitored")
public boolean getMonitorThreads()
{
return !getBeans(ConnectorsThreadPoolLowResourceCheck.class).isEmpty();
}
/**
* @param monitorThreads If true, check connectors executors to see if they are
* {@link ThreadPool} instances that are low on threads.
*/
public void setMonitorThreads(boolean monitorThreads)
{
if(monitorThreads)
// already configured?
if ( !getMonitorThreads() ) addLowResourceCheck( new ConnectorsThreadPoolLowResourceCheck() );
else
getBeans(ConnectorsThreadPoolLowResourceCheck.class).forEach(this::removeBean);
}
/**
* @return The maximum connections allowed for the monitored connectors before low resource handling is activated
* @deprecated Replaced by ConnectionLimit
*/
@ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated")
@Deprecated
public int getMaxConnections()
{
for(MaxConnectionsLowResourceCheck lowResourceCheck : getBeans(MaxConnectionsLowResourceCheck.class))
{
if (lowResourceCheck.getMaxConnections()>0)
{
return lowResourceCheck.getMaxConnections();
}
}
return -1;
}
/**
* @param maxConnections The maximum connections before low resources state is triggered
* @deprecated Replaced by ConnectionLimit
*/
@Deprecated
public void setMaxConnections(int maxConnections)
{
if (maxConnections>0)
{
if (getBeans(MaxConnectionsLowResourceCheck.class).isEmpty())
{
addLowResourceCheck(new MaxConnectionsLowResourceCheck(maxConnections));
} else
{
getBeans(MaxConnectionsLowResourceCheck.class).forEach( c -> c.setMaxConnections( maxConnections ) );
}
}
else
{
getBeans(ConnectorsThreadPoolLowResourceCheck.class).forEach(this::removeBean);
}
}
@ManagedAttribute("The reasons the monitored connectors are low on resources")
public String getReasons()
{
return _reasons;
}
protected void setReasons(String reasons)
{
_reasons = reasons;
}
@ManagedAttribute("Are the monitored connectors low on resources?")
@ -111,24 +174,39 @@ public class LowResourceMonitor extends AbstractLifeCycle
return _low.get();
}
protected boolean enableLowOnResources(boolean expectedValue, boolean newValue)
{
return _low.compareAndSet(expectedValue, newValue);
}
@ManagedAttribute("The reason(s) the monitored connectors are low on resources")
public String getLowResourcesReasons()
{
return _reasons;
}
protected void setLowResourcesReasons(String reasons)
{
_reasons = reasons;
}
@ManagedAttribute("Get the timestamp in ms since epoch that low resources state started")
public long getLowResourcesStarted()
{
return _lowStarted;
}
public void setLowResourcesStarted(long lowStarted)
{
_lowStarted = lowStarted;
}
@ManagedAttribute("The monitored connectors. If null then all server connectors are monitored")
public Collection<Connector> getMonitoredConnectors()
{
if (_monitoredConnectors==null)
return Collections.emptyList();
return Arrays.asList(_monitoredConnectors);
return Arrays.asList( _monitoredConnectors);
}
/**
@ -142,6 +220,13 @@ public class LowResourceMonitor extends AbstractLifeCycle
_monitoredConnectors = monitoredConnectors.toArray(new Connector[monitoredConnectors.size()]);
}
protected Connector[] getMonitoredOrServerConnectors()
{
if (_monitoredConnectors!=null && _monitoredConnectors.length>0)
return _monitoredConnectors;
return _server.getConnectors();
}
@ManagedAttribute("If false, new connections are not accepted while in low resources")
public boolean isAcceptingInLowResources()
{
@ -152,7 +237,7 @@ public class LowResourceMonitor extends AbstractLifeCycle
{
_acceptingInLowResources = acceptingInLowResources;
}
@ManagedAttribute("The monitor period in ms")
public int getPeriod()
{
@ -167,58 +252,6 @@ public class LowResourceMonitor extends AbstractLifeCycle
_period = periodMS;
}
@ManagedAttribute("True if low available threads status is monitored")
public boolean getMonitorThreads()
{
return _monitorThreads;
}
/**
* @param monitorThreads If true, check connectors executors to see if they are
* {@link ThreadPool} instances that are low on threads.
*/
public void setMonitorThreads(boolean monitorThreads)
{
_monitorThreads = monitorThreads;
}
/**
* @return The maximum connections allowed for the monitored connectors before low resource handling is activated
* @deprecated Replaced by ConnectionLimit
*/
@ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated")
@Deprecated
public int getMaxConnections()
{
return _maxConnections;
}
/**
* @param maxConnections The maximum connections before low resources state is triggered
* @deprecated Replaced by ConnectionLimit
*/
@Deprecated
public void setMaxConnections(int maxConnections)
{
if (maxConnections>0)
LOG.warn("LowResourceMonitor.setMaxConnections is deprecated. Use ConnectionLimit.");
_maxConnections = maxConnections;
}
@ManagedAttribute("The maximum memory (in bytes) that can be used before low resources is triggered. Memory used is calculated as (totalMemory-freeMemory).")
public long getMaxMemory()
{
return _maxMemory;
}
/**
* @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered.
*/
public void setMaxMemory(long maxMemoryBytes)
{
_maxMemory = maxMemoryBytes;
}
@ManagedAttribute("The idletimeout in ms to apply to all existing connections when low resources is detected")
public int getLowResourcesIdleTimeout()
{
@ -247,6 +280,99 @@ public class LowResourceMonitor extends AbstractLifeCycle
_maxLowResourcesTime = maxLowResourcesTimeMS;
}
@ManagedAttribute("The maximum memory (in bytes) that can be used before low resources is triggered. Memory used is calculated as (totalMemory-freeMemory).")
public long getMaxMemory()
{
Collection<MemoryLowResourceCheck> beans = getBeans(MemoryLowResourceCheck.class);
if(beans.isEmpty())
{
return 0;
}
return beans.stream().findFirst().get().getMaxMemory();
}
/**
* @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered.
*/
public void setMaxMemory(long maxMemoryBytes)
{
if(maxMemoryBytes<=0)
{
return;
}
Collection<MemoryLowResourceCheck> beans = getBeans(MemoryLowResourceCheck.class);
if(beans.isEmpty())
addLowResourceCheck( new MemoryLowResourceCheck( maxMemoryBytes ) );
else
beans.forEach( lowResourceCheck -> lowResourceCheck.setMaxMemory( maxMemoryBytes ) );
}
public Set<LowResourceCheck> getLowResourceChecks()
{
return _lowResourceChecks;
}
public void setLowResourceChecks( Set<LowResourceCheck> lowResourceChecks )
{
updateBeans(_lowResourceChecks.toArray(),lowResourceChecks.toArray());
this._lowResourceChecks = lowResourceChecks;
}
public void addLowResourceCheck( LowResourceCheck lowResourceCheck )
{
addBean(lowResourceCheck);
this._lowResourceChecks.add(lowResourceCheck);
}
protected void monitor()
{
String reasons=null;
for(LowResourceCheck lowResourceCheck : _lowResourceChecks)
{
if(lowResourceCheck.isLowOnResources())
{
reasons = lowResourceCheck.toString();
break;
}
}
if (reasons!=null)
{
// Log the reasons if there is any change in the cause
if (!reasons.equals(getReasons()))
{
LOG.warn("Low Resources: {}",reasons);
setReasons(reasons);
}
// Enter low resources state?
if (enableLowOnResources(false,true))
{
setLowResourcesReasons(reasons);
setLowResourcesStarted(System.currentTimeMillis());
setLowResources();
}
// Too long in low resources state?
if ( getMaxLowResourcesTime()>0 && (System.currentTimeMillis()-getLowResourcesStarted())>getMaxLowResourcesTime())
setLowResources();
}
else
{
if (enableLowOnResources(true,false))
{
LOG.info("Low Resources cleared");
setLowResourcesReasons(null);
setLowResourcesStarted(0);
setReasons(null);
clearLowResources();
}
}
}
@Override
protected void doStart() throws Exception
{
@ -257,6 +383,7 @@ public class LowResourceMonitor extends AbstractLifeCycle
_scheduler=new LRMScheduler();
_scheduler.start();
}
super.doStart();
_scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
@ -265,94 +392,11 @@ public class LowResourceMonitor extends AbstractLifeCycle
@Override
protected void doStop() throws Exception
{
if (_scheduler instanceof LRMScheduler)
if (_scheduler instanceof LRMScheduler )
_scheduler.stop();
super.doStop();
}
protected Connector[] getMonitoredOrServerConnectors()
{
if (_monitoredConnectors!=null && _monitoredConnectors.length>0)
return _monitoredConnectors;
return _server.getConnectors();
}
protected void monitor()
{
String reasons=null;
String cause="";
int connections=0;
ThreadPool serverThreads = _server.getThreadPool();
if (_monitorThreads && serverThreads.isLowOnThreads())
{
reasons=low(reasons,"Server low on threads: "+serverThreads);
cause+="S";
}
for(Connector connector : getMonitoredOrServerConnectors())
{
connections+=connector.getConnectedEndPoints().size();
Executor executor = connector.getExecutor();
if (executor instanceof ThreadPool && executor!=serverThreads)
{
ThreadPool connectorThreads=(ThreadPool)executor;
if (_monitorThreads && connectorThreads.isLowOnThreads())
{
reasons=low(reasons,"Connector low on threads: "+connectorThreads);
cause+="T";
}
}
}
if (_maxConnections>0 && connections>_maxConnections)
{
reasons=low(reasons,"Max Connections exceeded: "+connections+">"+_maxConnections);
cause+="C";
}
long memory=Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory();
if (_maxMemory>0 && memory>_maxMemory)
{
reasons=low(reasons,"Max memory exceeded: "+memory+">"+_maxMemory);
cause+="M";
}
if (reasons!=null)
{
// Log the reasons if there is any change in the cause
if (!cause.equals(_cause))
{
LOG.warn("Low Resources: {}",reasons);
_cause=cause;
}
// Enter low resources state?
if (_low.compareAndSet(false,true))
{
_reasons=reasons;
_lowStarted=System.currentTimeMillis();
setLowResources();
}
// Too long in low resources state?
if (_maxLowResourcesTime>0 && (System.currentTimeMillis()-_lowStarted)>_maxLowResourcesTime)
setLowResources();
}
else
{
if (_low.compareAndSet(true,false))
{
LOG.info("Low Resources cleared");
_reasons=null;
_lowStarted=0;
_cause=null;
clearLowResources();
}
}
}
protected void setLowResources()
{
for(Connector connector : getMonitoredOrServerConnectors())
@ -366,8 +410,8 @@ public class LowResourceMonitor extends AbstractLifeCycle
c.setAccepting(false);
}
}
for (EndPoint endPoint : connector.getConnectedEndPoints())
for ( EndPoint endPoint : connector.getConnectedEndPoints())
endPoint.setIdleTimeout(_lowResourcesIdleTimeout);
}
}
@ -379,7 +423,7 @@ public class LowResourceMonitor extends AbstractLifeCycle
for (EndPoint endPoint : connector.getConnectedEndPoints())
endPoint.setIdleTimeout(connector.getIdleTimeout());
}
for (AbstractConnector connector : _acceptingConnectors)
{
connector.setAccepting(true);
@ -387,15 +431,219 @@ public class LowResourceMonitor extends AbstractLifeCycle
_acceptingConnectors.clear();
}
private String low(String reasons, String newReason)
protected String low(String reasons, String newReason)
{
if (reasons==null)
return newReason;
return reasons+", "+newReason;
}
private static class LRMScheduler extends ScheduledExecutorScheduler
{
}
interface LowResourceCheck
{
boolean isLowOnResources();
String getReason();
}
//------------------------------------------------------
// default implementations for backward compat
//------------------------------------------------------
public class MainThreadPoolLowResourceCheck implements LowResourceCheck
{
private String reason;
public MainThreadPoolLowResourceCheck()
{
// no op
}
@Override
public boolean isLowOnResources()
{
ThreadPool serverThreads = _server.getThreadPool();
if (serverThreads.isLowOnThreads())
{
reason="Server low on threads: "+serverThreads;
return true;
}
return false;
}
@Override
public String getReason()
{
return reason;
}
@Override
public String toString()
{
return "Check if the server ThreadPool is lowOnThreads";
}
}
public class ConnectorsThreadPoolLowResourceCheck implements LowResourceCheck
{
private String reason;
public ConnectorsThreadPoolLowResourceCheck()
{
// no op
}
@Override
public boolean isLowOnResources()
{
ThreadPool serverThreads = _server.getThreadPool();
if(serverThreads.isLowOnThreads())
{
reason ="Server low on threads: "+serverThreads.getThreads()+", idleThreads:"+serverThreads.getIdleThreads();
return true;
}
for(Connector connector : getMonitoredConnectors())
{
Executor executor = connector.getExecutor();
if (executor instanceof ThreadPool && executor!=serverThreads)
{
ThreadPool connectorThreads=(ThreadPool)executor;
if (connectorThreads.isLowOnThreads())
{
reason ="Connector low on threads: "+connectorThreads;
return true;
}
}
}
return false;
}
@Override
public String getReason()
{
return reason;
}
@Override
public String toString()
{
return "Check if the ThreadPool from monitored connectors are lowOnThreads and if all connections number is higher than the allowed maxConnection";
}
}
@ManagedObject("Check max allowed connections on connectors")
public class MaxConnectionsLowResourceCheck implements LowResourceCheck
{
private String reason;
private int maxConnections;
public MaxConnectionsLowResourceCheck(int maxConnections)
{
this.maxConnections = maxConnections;
}
/**
* @return The maximum connections allowed for the monitored connectors before low resource handling is activated
* @deprecated Replaced by ConnectionLimit
*/
@ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated")
@Deprecated
public int getMaxConnections()
{
return maxConnections;
}
/**
* @param maxConnections The maximum connections before low resources state is triggered
* @deprecated Replaced by ConnectionLimit
*/
@Deprecated
public void setMaxConnections(int maxConnections)
{
if (maxConnections>0)
LOG.warn("LowResourceMonitor.setMaxConnections is deprecated. Use ConnectionLimit.");
this.maxConnections = maxConnections;
}
@Override
public boolean isLowOnResources()
{
int connections=0;
for(Connector connector : getMonitoredConnectors())
{
connections+=connector.getConnectedEndPoints().size();
}
if (maxConnections>0 && connections>maxConnections)
{
reason ="Max Connections exceeded: "+connections+">"+maxConnections;
return true;
}
return false;
}
@Override
public String getReason()
{
return reason;
}
@Override
public String toString()
{
return "All connections number is higher than the allowed maxConnection";
}
}
public class MemoryLowResourceCheck implements LowResourceCheck
{
private String reason;
private long maxMemory;
public MemoryLowResourceCheck(long maxMemory)
{
this.maxMemory = maxMemory;
}
@Override
public boolean isLowOnResources()
{
long memory=Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory();
if (maxMemory>0 && memory>maxMemory)
{
reason = "Max memory exceeded: "+memory+">"+maxMemory;
return true;
}
return false;
}
public long getMaxMemory()
{
return maxMemory;
}
/**
* @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered.
*/
public void setMaxMemory( long maxMemoryBytes )
{
this.maxMemory = maxMemoryBytes;
}
@Override
public String getReason()
{
return reason;
}
@Override
public String toString()
{
return "Check if used memory is higher than the allowed max memory";
}
}
}

194
jetty-server/src/test/java/org/eclipse/jetty/server/CustomResourcesMonitorTest.java Normal file
View File

@ -0,0 +1,194 @@
//
// ========================================================================
// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.server;
import org.eclipse.jetty.toolchain.test.AdvancedRunner;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.log.Logger;
import org.eclipse.jetty.util.thread.TimerScheduler;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import java.io.IOException;
import java.io.InputStream;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.Assert.*;
@RunWith(AdvancedRunner.class)
public class CustomResourcesMonitorTest
{
Server _server;
ServerConnector _connector;
FileOnDirectoryMonitor _fileOnDirectoryMonitor;
Path _monitoredPath;
LowResourceMonitor _lowResourceMonitor;
@Before
public void before() throws Exception
{
_server = new Server();
_server.addBean(new TimerScheduler());
_connector = new ServerConnector(_server);
_connector.setPort(0);
_connector.setIdleTimeout(35000);
_server.addConnector(_connector);
_server.setHandler(new DumpHandler());
_monitoredPath = Files.createTempDirectory( "jetty_test" );
_fileOnDirectoryMonitor=new FileOnDirectoryMonitor(_monitoredPath);
_lowResourceMonitor = new LowResourceMonitor(_server);
_server.addBean( _lowResourceMonitor );
_lowResourceMonitor.addLowResourceCheck( _fileOnDirectoryMonitor );
_server.start();
}
@After
public void after() throws Exception
{
_server.stop();
}
@Test
public void testFileOnDirectoryMonitor() throws Exception
{
int monitorPeriod = _lowResourceMonitor.getPeriod();
int lowResourcesIdleTimeout = _lowResourceMonitor.getLowResourcesIdleTimeout();
assertThat(lowResourcesIdleTimeout, Matchers.lessThanOrEqualTo(monitorPeriod));
int maxLowResourcesTime = 5 * monitorPeriod;
_lowResourceMonitor.setMaxLowResourcesTime(maxLowResourcesTime);
assertFalse(_fileOnDirectoryMonitor.isLowOnResources());
try(Socket socket0 = new Socket("localhost",_connector.getLocalPort()))
{
Path tmpFile = Files.createTempFile( _monitoredPath, "yup", ".tmp" );
// Write a file
Files.write(tmpFile, "foobar".getBytes());
// Wait a couple of monitor periods so that
// fileOnDirectoryMonitor detects it is in low mode.
Thread.sleep(2 * monitorPeriod);
assertTrue(_fileOnDirectoryMonitor.isLowOnResources());
// We already waited enough for fileOnDirectoryMonitor to close socket0.
assertEquals(-1, socket0.getInputStream().read());
// New connections are not affected by the
// low mode until maxLowResourcesTime elapses.
try(Socket socket1 = new Socket("localhost",_connector.getLocalPort()))
{
// Set a very short read timeout so we can test if the server closed.
socket1.setSoTimeout(1);
InputStream input1 = socket1.getInputStream();
assertTrue(_fileOnDirectoryMonitor.isLowOnResources());
try
{
input1.read();
fail();
}
catch (SocketTimeoutException expected)
{
}
// Wait a couple of lowResources idleTimeouts.
Thread.sleep(2 * lowResourcesIdleTimeout);
// Verify the new socket is still open.
assertTrue(_fileOnDirectoryMonitor.isLowOnResources());
try
{
input1.read();
fail();
}
catch (SocketTimeoutException expected)
{
}
Files.delete( tmpFile );
// Let the maxLowResourcesTime elapse.
Thread.sleep(maxLowResourcesTime);
assertFalse(_fileOnDirectoryMonitor.isLowOnResources());
}
}
}
static class FileOnDirectoryMonitor implements LowResourceMonitor.LowResourceCheck
{
private static final Logger LOG = Log.getLogger( FileOnDirectoryMonitor.class);
private final Path _pathToMonitor;
private String reason;
public FileOnDirectoryMonitor(Path pathToMonitor )
{
_pathToMonitor = pathToMonitor;
}
@Override
public boolean isLowOnResources()
{
try
{
Stream<Path> paths = Files.list( _pathToMonitor );
List<Path> content = paths.collect( Collectors.toList() );
if(!content.isEmpty())
{
reason= "directory not empty so enable low resources";
return true;
}
}
catch ( IOException e )
{
LOG.info( "ignore issue looking at directory content", e );
}
return false;
}
@Override
public String getReason()
{
return reason;
}
@Override
public String toString()
{
return getClass().getName();
}
}
}

113
jetty-server/src/test/java/org/eclipse/jetty/server/LowResourcesMonitorTest.java
View File

@ -19,27 +19,27 @@
package org.eclipse.jetty.server;
import java.io.InputStream;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import org.eclipse.jetty.toolchain.test.AdvancedRunner;
import org.eclipse.jetty.util.thread.QueuedThreadPool;
import org.eclipse.jetty.util.thread.TimerScheduler;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import java.io.InputStream;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.CountDownLatch;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.*;
@RunWith(AdvancedRunner.class)
public class LowResourcesMonitorTest
@ -48,7 +48,7 @@ public class LowResourcesMonitorTest
Server _server;
ServerConnector _connector;
LowResourceMonitor _lowResourcesMonitor;
@Before
public void before() throws Exception
{
@ -71,6 +71,7 @@ public class LowResourcesMonitorTest
_lowResourcesMonitor.setLowResourcesIdleTimeout(200);
_lowResourcesMonitor.setMaxConnections(20);
_lowResourcesMonitor.setPeriod(900);
_lowResourcesMonitor.setMonitoredConnectors( Collections.singleton( _connector ) );
_server.addBean(_lowResourcesMonitor);
_server.start();
@ -86,38 +87,35 @@ public class LowResourcesMonitorTest
@Test
public void testLowOnThreads() throws Exception
{
_lowResourcesMonitor.setMonitorThreads(true);
Thread.sleep(1200);
_threadPool.setMaxThreads(_threadPool.getThreads()-_threadPool.getIdleThreads()+10);
Thread.sleep(1200);
Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
assertFalse(_lowResourcesMonitor.getReasons(), _lowResourcesMonitor.isLowOnResources());
final CountDownLatch latch = new CountDownLatch(1);
for (int i=0;i<100;i++)
{
_threadPool.execute(new Runnable()
_threadPool.execute(() ->
{
@Override
public void run()
try
{
try
{
latch.await();
}
catch (InterruptedException e)
{
e.printStackTrace();
}
latch.await();
}
catch (InterruptedException e)
{
e.printStackTrace();
}
});
}
Thread.sleep(1200);
Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
assertTrue(_lowResourcesMonitor.isLowOnResources());
latch.countDown();
Thread.sleep(1200);
Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
}
@ -125,10 +123,13 @@ public class LowResourcesMonitorTest
public void testNotAccepting() throws Exception
{
_lowResourcesMonitor.setAcceptingInLowResources(false);
_lowResourcesMonitor.setMonitorThreads( true );
Thread.sleep(1200);
_threadPool.setMaxThreads(_threadPool.getThreads()-_threadPool.getIdleThreads()+10);
int maxThreads = _threadPool.getThreads()-_threadPool.getIdleThreads()+10;
System.out.println("maxThreads:"+maxThreads);
_threadPool.setMaxThreads(maxThreads);
Thread.sleep(1200);
Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
for (AbstractConnector c : _server.getBeans(AbstractConnector.class))
assertThat(c.isAccepting(),Matchers.is(true));
@ -136,31 +137,27 @@ public class LowResourcesMonitorTest
final CountDownLatch latch = new CountDownLatch(1);
for (int i=0;i<100;i++)
{
_threadPool.execute(new Runnable()
_threadPool.execute(() ->
{
@Override
public void run()
try
{
try
{
latch.await();
}
catch (InterruptedException e)
{
e.printStackTrace();
}
latch.await();
}
catch (InterruptedException e)
{
e.printStackTrace();
}
});
}
Thread.sleep(1200);
Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
assertTrue(_lowResourcesMonitor.isLowOnResources());
for (AbstractConnector c : _server.getBeans(AbstractConnector.class))
assertThat(c.isAccepting(),Matchers.is(false));
latch.countDown();
Thread.sleep(1200);
Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
for (AbstractConnector c : _server.getBeans(AbstractConnector.class))
assertThat(c.isAccepting(),Matchers.is(true));
}
@ -172,7 +169,7 @@ public class LowResourcesMonitorTest
{
_lowResourcesMonitor.setMaxMemory(Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory()+(100*1024*1024));
Thread.sleep(1200);
Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
byte[] data = new byte[100*1024*1024];
Arrays.fill(data,(byte)1);
@ -180,13 +177,13 @@ public class LowResourcesMonitorTest
assertThat(hash,not(equalTo(0)));
Thread.sleep(1200);
Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
assertTrue(_lowResourcesMonitor.isLowOnResources());
data=null;
System.gc();
System.gc();
Thread.sleep(1200);
Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
}
@ -194,26 +191,27 @@ public class LowResourcesMonitorTest
public void testMaxConnectionsAndMaxIdleTime() throws Exception
{
_lowResourcesMonitor.setMaxMemory(0);
Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
assertEquals( 20, _lowResourcesMonitor.getMaxConnections() );
Socket[] socket = new Socket[_lowResourcesMonitor.getMaxConnections()+1];
for (int i=0;i<socket.length;i++)
socket[i]=new Socket("localhost",_connector.getLocalPort());
Thread.sleep(1200);
Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
assertTrue(_lowResourcesMonitor.isLowOnResources());
try(Socket newSocket = new Socket("localhost",_connector.getLocalPort()))
{
// wait for low idle time to close sockets, but not new Socket
Thread.sleep(1200);
Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
for (int i=0;i<socket.length;i++)
Assert.assertEquals(-1,socket[i].getInputStream().read());
assertEquals(-1,socket[i].getInputStream().read());
newSocket.getOutputStream().write("GET / HTTP/1.0\r\n\r\n".getBytes(StandardCharsets.UTF_8));
Assert.assertEquals('H',newSocket.getInputStream().read());
assertEquals('H',newSocket.getInputStream().read());
}
}
@ -222,11 +220,11 @@ public class LowResourcesMonitorTest
{
int monitorPeriod = _lowResourcesMonitor.getPeriod();
int lowResourcesIdleTimeout = _lowResourcesMonitor.getLowResourcesIdleTimeout();
Assert.assertThat(lowResourcesIdleTimeout, Matchers.lessThan(monitorPeriod));
assertThat(lowResourcesIdleTimeout, Matchers.lessThan(monitorPeriod));
int maxLowResourcesTime = 5 * monitorPeriod;
_lowResourcesMonitor.setMaxLowResourcesTime(maxLowResourcesTime);
Assert.assertFalse(_lowResourcesMonitor.isLowOnResources());
assertFalse(_lowResourcesMonitor.getReasons(),_lowResourcesMonitor.isLowOnResources());
try(Socket socket0 = new Socket("localhost",_connector.getLocalPort()))
{
@ -236,10 +234,10 @@ public class LowResourcesMonitorTest
// Wait a couple of monitor periods so that
// lowResourceMonitor detects it is in low mode.
Thread.sleep(2 * monitorPeriod);
Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
assertTrue(_lowResourcesMonitor.isLowOnResources());
// We already waited enough for lowResourceMonitor to close socket0.
Assert.assertEquals(-1, socket0.getInputStream().read());
assertEquals(-1, socket0.getInputStream().read());
// New connections are not affected by the
// low mode until maxLowResourcesTime elapses.
@ -249,11 +247,11 @@ public class LowResourcesMonitorTest
socket1.setSoTimeout(1);
InputStream input1 = socket1.getInputStream();
Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
assertTrue(_lowResourcesMonitor.isLowOnResources());
try
{
input1.read();
Assert.fail();
fail();
}
catch (SocketTimeoutException expected)
{
@ -263,22 +261,21 @@ public class LowResourcesMonitorTest
Thread.sleep(2 * lowResourcesIdleTimeout);
// Verify the new socket is still open.
Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
assertTrue(_lowResourcesMonitor.isLowOnResources());
try
{
input1.read();
Assert.fail();
fail();
}
catch (SocketTimeoutException expected)
{
}
// Let the maxLowResourcesTime elapse.
Thread.sleep(maxLowResourcesTime);
Assert.assertTrue(_lowResourcesMonitor.isLowOnResources());
assertTrue(_lowResourcesMonitor.isLowOnResources());
// Now also the new socket should be closed.
Assert.assertEquals(-1, input1.read());
assertEquals(-1, input1.read());
}
}
}