Issue #5539 - Proper StatisticsServlet output format via content negotiation
Signed-off-by: Joakim Erdfelt <joakim.erdfelt@gmail.com>
This commit is contained in:
parent
3cbd117ec9
commit
5362467e67
|
@ -41,6 +41,11 @@
|
|||
<artifactId>jetty-security</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-util-ajax</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-jmx</artifactId>
|
||||
|
|
|
@ -19,29 +19,41 @@
|
|||
package org.eclipse.jetty.servlet;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.lang.management.MemoryMXBean;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import javax.servlet.ServletContext;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.eclipse.jetty.http.QuotedQualityCSV;
|
||||
import org.eclipse.jetty.io.ConnectionStatistics;
|
||||
import org.eclipse.jetty.server.AbstractConnector;
|
||||
import org.eclipse.jetty.server.Connector;
|
||||
import org.eclipse.jetty.server.ConnectorStatistics;
|
||||
import org.eclipse.jetty.server.Handler;
|
||||
import org.eclipse.jetty.server.Server;
|
||||
import org.eclipse.jetty.server.handler.ContextHandler;
|
||||
import org.eclipse.jetty.server.handler.StatisticsHandler;
|
||||
import org.eclipse.jetty.util.component.Container;
|
||||
import org.eclipse.jetty.util.StringUtil;
|
||||
import org.eclipse.jetty.util.ajax.JSON;
|
||||
import org.eclipse.jetty.util.log.Log;
|
||||
import org.eclipse.jetty.util.log.Logger;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
public class StatisticsServlet extends HttpServlet
|
||||
{
|
||||
private static final Logger LOG = Log.getLogger(StatisticsServlet.class);
|
||||
|
@ -49,7 +61,7 @@ public class StatisticsServlet extends HttpServlet
|
|||
boolean _restrictToLocalhost = true; // defaults to true
|
||||
private StatisticsHandler _statsHandler;
|
||||
private MemoryMXBean _memoryBean;
|
||||
private Connector[] _connectors;
|
||||
private List<Connector> _connectors;
|
||||
|
||||
@Override
|
||||
public void init() throws ServletException
|
||||
|
@ -58,20 +70,16 @@ public class StatisticsServlet extends HttpServlet
|
|||
ContextHandler.Context scontext = (ContextHandler.Context)context;
|
||||
Server server = scontext.getContextHandler().getServer();
|
||||
|
||||
Handler handler = server.getChildHandlerByClass(StatisticsHandler.class);
|
||||
_statsHandler = server.getChildHandlerByClass(StatisticsHandler.class);
|
||||
|
||||
if (handler != null)
|
||||
{
|
||||
_statsHandler = (StatisticsHandler)handler;
|
||||
}
|
||||
else
|
||||
if (_statsHandler == null)
|
||||
{
|
||||
LOG.warn("Statistics Handler not installed!");
|
||||
return;
|
||||
}
|
||||
|
||||
_memoryBean = ManagementFactory.getMemoryMXBean();
|
||||
_connectors = server.getConnectors();
|
||||
_connectors = Arrays.asList(server.getConnectors());
|
||||
|
||||
if (getInitParameter("restrictToLocalhost") != null)
|
||||
{
|
||||
|
@ -80,47 +88,150 @@ public class StatisticsServlet extends HttpServlet
|
|||
}
|
||||
|
||||
@Override
|
||||
public void doPost(HttpServletRequest sreq, HttpServletResponse sres) throws ServletException, IOException
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
|
||||
{
|
||||
doGet(sreq, sres);
|
||||
doGet(request, response);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException
|
||||
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
|
||||
{
|
||||
if (_statsHandler == null)
|
||||
{
|
||||
LOG.warn("Statistics Handler not installed!");
|
||||
resp.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
|
||||
response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
|
||||
return;
|
||||
}
|
||||
if (_restrictToLocalhost)
|
||||
{
|
||||
if (!isLoopbackAddress(req.getRemoteAddr()))
|
||||
if (!isLoopbackAddress(request.getRemoteAddr()))
|
||||
{
|
||||
resp.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
|
||||
response.sendError(HttpServletResponse.SC_FORBIDDEN);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (Boolean.parseBoolean(req.getParameter("statsReset")))
|
||||
if (Boolean.parseBoolean(request.getParameter("statsReset")))
|
||||
{
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
_statsHandler.statsReset();
|
||||
return;
|
||||
}
|
||||
|
||||
String wantXml = req.getParameter("xml");
|
||||
if (wantXml == null)
|
||||
wantXml = req.getParameter("XML");
|
||||
List<String> acceptable = getOrderedAcceptableMimeTypes(request);
|
||||
|
||||
if (Boolean.parseBoolean(wantXml))
|
||||
for (String mimeType : acceptable)
|
||||
{
|
||||
sendXmlResponse(resp);
|
||||
switch (mimeType)
|
||||
{
|
||||
case "application/json":
|
||||
writeJsonResponse(response);
|
||||
return;
|
||||
case "text/xml":
|
||||
writeXmlResponse(response);
|
||||
return;
|
||||
case "text/html":
|
||||
writeHtmlResponse(response);
|
||||
return;
|
||||
case "text/plain":
|
||||
writeTextResponse(response);
|
||||
return;
|
||||
case "*/*":
|
||||
String wantXml = request.getParameter("xml");
|
||||
if (wantXml == null)
|
||||
wantXml = request.getParameter("XML");
|
||||
|
||||
if (Boolean.parseBoolean(wantXml))
|
||||
{
|
||||
writeXmlResponse(response);
|
||||
}
|
||||
else
|
||||
{
|
||||
writeTextResponse(response);
|
||||
}
|
||||
return;
|
||||
default:
|
||||
if (LOG.isDebugEnabled())
|
||||
{
|
||||
LOG.debug("Ignoring unrecognized mime-type {}", mimeType);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
// None of the listed `Accept` mime-types were found.
|
||||
response.sendError(HttpServletResponse.SC_NOT_ACCEPTABLE);
|
||||
}
|
||||
|
||||
private void writeTextResponse(HttpServletResponse response) throws IOException
|
||||
{
|
||||
response.setCharacterEncoding("utf-8");
|
||||
response.setContentType("text/plain");
|
||||
CharSequence text = generateResponse(new TextProducer());
|
||||
response.getWriter().print(text.toString());
|
||||
}
|
||||
|
||||
private void writeHtmlResponse(HttpServletResponse response) throws IOException
|
||||
{
|
||||
response.setCharacterEncoding("utf-8");
|
||||
response.setContentType("text/html");
|
||||
Writer htmlWriter = new OutputStreamWriter(response.getOutputStream(), UTF_8);
|
||||
htmlWriter.append("<html><head><title>");
|
||||
htmlWriter.append(this.getClass().getSimpleName());
|
||||
htmlWriter.append("</title></head><body>\n");
|
||||
CharSequence html = generateResponse(new HtmlProducer());
|
||||
htmlWriter.append(html.toString());
|
||||
htmlWriter.append("\n</body></html>\n");
|
||||
htmlWriter.flush();
|
||||
}
|
||||
|
||||
private void writeXmlResponse(HttpServletResponse response) throws IOException
|
||||
{
|
||||
response.setCharacterEncoding("utf-8");
|
||||
response.setContentType("text/xml");
|
||||
CharSequence xml = generateResponse(new XmlProducer());
|
||||
response.getWriter().print(xml.toString());
|
||||
}
|
||||
|
||||
private void writeJsonResponse(HttpServletResponse response) throws IOException
|
||||
{
|
||||
// We intentionally don't put "UTF-8" into the response headers
|
||||
// as the rules for application/json state that it should never be
|
||||
// present on the HTTP Content-Type header.
|
||||
// It is also true that the application/json mime-type is always UTF-8.
|
||||
response.setContentType("application/json");
|
||||
CharSequence json = generateResponse(new JsonProducer());
|
||||
Writer jsonWriter = new OutputStreamWriter(response.getOutputStream(), UTF_8);
|
||||
jsonWriter.append(json);
|
||||
jsonWriter.flush();
|
||||
}
|
||||
|
||||
private List<String> getOrderedAcceptableMimeTypes(HttpServletRequest request)
|
||||
{
|
||||
QuotedQualityCSV values = null;
|
||||
Enumeration<String> enumAccept = request.getHeaders("Accept");
|
||||
if (enumAccept != null)
|
||||
{
|
||||
sendTextResponse(resp);
|
||||
while (enumAccept.hasMoreElements())
|
||||
{
|
||||
String value = enumAccept.nextElement();
|
||||
if (StringUtil.isNotBlank(value))
|
||||
{
|
||||
if (values == null)
|
||||
{
|
||||
values = new QuotedQualityCSV(QuotedQualityCSV.MOST_SPECIFIC_MIME_ORDERING);
|
||||
}
|
||||
values.addValue(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (values != null)
|
||||
{
|
||||
return values.getValues();
|
||||
}
|
||||
|
||||
// No accept specified, return that we allow ALL mime types
|
||||
return Collections.singletonList("*/*");
|
||||
}
|
||||
|
||||
private boolean isLoopbackAddress(String address)
|
||||
|
@ -137,177 +248,362 @@ public class StatisticsServlet extends HttpServlet
|
|||
}
|
||||
}
|
||||
|
||||
private void sendXmlResponse(HttpServletResponse response) throws IOException
|
||||
private CharSequence generateResponse(OutputProducer outputProducer)
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
Map<String, Object> top = new HashMap<>();
|
||||
|
||||
sb.append("<statistics>\n");
|
||||
// requests
|
||||
Map<String, Number> requests = new HashMap<>();
|
||||
requests.put("statsOnMs", _statsHandler.getStatsOnMs());
|
||||
|
||||
sb.append(" <requests>\n");
|
||||
sb.append(" <statsOnMs>").append(_statsHandler.getStatsOnMs()).append("</statsOnMs>\n");
|
||||
requests.put("requests", _statsHandler.getRequests());
|
||||
|
||||
sb.append(" <requests>").append(_statsHandler.getRequests()).append("</requests>\n");
|
||||
sb.append(" <requestsActive>").append(_statsHandler.getRequestsActive()).append("</requestsActive>\n");
|
||||
sb.append(" <requestsActiveMax>").append(_statsHandler.getRequestsActiveMax()).append("</requestsActiveMax>\n");
|
||||
sb.append(" <requestsTimeTotal>").append(_statsHandler.getRequestTimeTotal()).append("</requestsTimeTotal>\n");
|
||||
sb.append(" <requestsTimeMean>").append(_statsHandler.getRequestTimeMean()).append("</requestsTimeMean>\n");
|
||||
sb.append(" <requestsTimeMax>").append(_statsHandler.getRequestTimeMax()).append("</requestsTimeMax>\n");
|
||||
sb.append(" <requestsTimeStdDev>").append(_statsHandler.getRequestTimeStdDev()).append("</requestsTimeStdDev>\n");
|
||||
requests.put("requestsActive", _statsHandler.getRequestsActive());
|
||||
requests.put("requestsActiveMax", _statsHandler.getRequestsActiveMax());
|
||||
requests.put("requestsTimeTotal", _statsHandler.getRequestTimeTotal());
|
||||
requests.put("requestsTimeMean", _statsHandler.getRequestTimeMean());
|
||||
requests.put("requestsTimeMax", _statsHandler.getRequestTimeMax());
|
||||
requests.put("requestsTimeStdDev", _statsHandler.getRequestTimeStdDev());
|
||||
|
||||
sb.append(" <dispatched>").append(_statsHandler.getDispatched()).append("</dispatched>\n");
|
||||
sb.append(" <dispatchedActive>").append(_statsHandler.getDispatchedActive()).append("</dispatchedActive>\n");
|
||||
sb.append(" <dispatchedActiveMax>").append(_statsHandler.getDispatchedActiveMax()).append("</dispatchedActiveMax>\n");
|
||||
sb.append(" <dispatchedTimeTotalMs>").append(_statsHandler.getDispatchedTimeTotal()).append("</dispatchedTimeTotalMs>\n");
|
||||
sb.append(" <dispatchedTimeMeanMs>").append(_statsHandler.getDispatchedTimeMean()).append("</dispatchedTimeMeanMs>\n");
|
||||
sb.append(" <dispatchedTimeMaxMs>").append(_statsHandler.getDispatchedTimeMax()).append("</dispatchedTimeMaxMs>\n");
|
||||
sb.append(" <dispatchedTimeStdDevMs>").append(_statsHandler.getDispatchedTimeStdDev()).append("</dispatchedTimeStdDevMs>\n");
|
||||
requests.put("dispatched", _statsHandler.getDispatched());
|
||||
requests.put("dispatchedActive", _statsHandler.getDispatchedActive());
|
||||
requests.put("dispatchedActiveMax", _statsHandler.getDispatchedActiveMax());
|
||||
requests.put("dispatchedTimeTotal", _statsHandler.getDispatchedTimeTotal());
|
||||
requests.put("dispatchedTimeMean", _statsHandler.getDispatchedTimeMean());
|
||||
requests.put("dispatchedTimeMax", _statsHandler.getDispatchedTimeMax());
|
||||
requests.put("dispatchedTimeStdDev", _statsHandler.getDispatchedTimeStdDev());
|
||||
|
||||
sb.append(" <asyncRequests>").append(_statsHandler.getAsyncRequests()).append("</asyncRequests>\n");
|
||||
sb.append(" <requestsSuspended>").append(_statsHandler.getAsyncRequestsWaiting()).append("</requestsSuspended>\n");
|
||||
sb.append(" <requestsSuspendedMax>").append(_statsHandler.getAsyncRequestsWaitingMax()).append("</requestsSuspendedMax>\n");
|
||||
sb.append(" <requestsResumed>").append(_statsHandler.getAsyncDispatches()).append("</requestsResumed>\n");
|
||||
sb.append(" <requestsExpired>").append(_statsHandler.getExpires()).append("</requestsExpired>\n");
|
||||
sb.append(" </requests>\n");
|
||||
requests.put("asyncRequests", _statsHandler.getAsyncRequests());
|
||||
requests.put("requestsSuspended", _statsHandler.getAsyncDispatches());
|
||||
requests.put("requestsSuspendedMax", _statsHandler.getAsyncRequestsWaiting());
|
||||
requests.put("requestsResumed", _statsHandler.getAsyncRequestsWaitingMax());
|
||||
requests.put("requestsExpired", _statsHandler.getExpires());
|
||||
|
||||
sb.append(" <responses>\n");
|
||||
sb.append(" <responses1xx>").append(_statsHandler.getResponses1xx()).append("</responses1xx>\n");
|
||||
sb.append(" <responses2xx>").append(_statsHandler.getResponses2xx()).append("</responses2xx>\n");
|
||||
sb.append(" <responses3xx>").append(_statsHandler.getResponses3xx()).append("</responses3xx>\n");
|
||||
sb.append(" <responses4xx>").append(_statsHandler.getResponses4xx()).append("</responses4xx>\n");
|
||||
sb.append(" <responses5xx>").append(_statsHandler.getResponses5xx()).append("</responses5xx>\n");
|
||||
sb.append(" <responsesBytesTotal>").append(_statsHandler.getResponsesBytesTotal()).append("</responsesBytesTotal>\n");
|
||||
sb.append(" </responses>\n");
|
||||
requests.put("errors", _statsHandler.getErrors());
|
||||
|
||||
sb.append(" <connections>\n");
|
||||
for (Connector connector : _connectors)
|
||||
top.put("requests", requests);
|
||||
|
||||
// responses
|
||||
Map<String, Number> responses = new HashMap<>();
|
||||
responses.put("responses1xx", _statsHandler.getResponses1xx());
|
||||
responses.put("responses2xx", _statsHandler.getResponses2xx());
|
||||
responses.put("responses3xx", _statsHandler.getResponses3xx());
|
||||
responses.put("responses4xx", _statsHandler.getResponses4xx());
|
||||
responses.put("responses5xx", _statsHandler.getResponses5xx());
|
||||
responses.put("responsesBytesTotal", _statsHandler.getResponsesBytesTotal());
|
||||
top.put("responses", responses);
|
||||
|
||||
// connections
|
||||
List<Object> connections = new ArrayList<>();
|
||||
_connectors.forEach((connector) ->
|
||||
{
|
||||
sb.append(" <connector>\n");
|
||||
sb.append(" <name>").append(connector.getClass().getName()).append("@").append(connector.hashCode()).append("</name>\n");
|
||||
sb.append(" <protocols>\n");
|
||||
for (String protocol : connector.getProtocols())
|
||||
{
|
||||
sb.append(" <protocol>").append(protocol).append("</protocol>\n");
|
||||
}
|
||||
sb.append(" </protocols>\n");
|
||||
Map<String, Object> connectorDetail = new HashMap<>();
|
||||
connectorDetail.put("name", String.format("%s@%X", connector.getClass().getName(), connector.hashCode()));
|
||||
connectorDetail.put("protocols", connector.getProtocols());
|
||||
|
||||
ConnectionStatistics connectionStats = null;
|
||||
if (connector instanceof AbstractConnector)
|
||||
connectionStats = connector.getBean(ConnectionStatistics.class);
|
||||
if (connectionStats != null)
|
||||
{
|
||||
sb.append(" <statsOn>true</statsOn>\n");
|
||||
sb.append(" <connections>").append(connectionStats.getConnectionsTotal()).append("</connections>\n");
|
||||
sb.append(" <connectionsOpen>").append(connectionStats.getConnections()).append("</connectionsOpen>\n");
|
||||
sb.append(" <connectionsOpenMax>").append(connectionStats.getConnectionsMax()).append("</connectionsOpenMax>\n");
|
||||
sb.append(" <connectionsDurationMean>").append(connectionStats.getConnectionDurationMean()).append("</connectionsDurationMean>\n");
|
||||
sb.append(" <connectionsDurationMax>").append(connectionStats.getConnectionDurationMax()).append("</connectionsDurationMax>\n");
|
||||
sb.append(" <connectionsDurationStdDev>").append(connectionStats.getConnectionDurationStdDev()).append("</connectionsDurationStdDev>\n");
|
||||
sb.append(" <bytesIn>").append(connectionStats.getReceivedBytes()).append("</bytesIn>\n");
|
||||
sb.append(" <bytesOut>").append(connectionStats.getSentBytes()).append("</bytesOut>\n");
|
||||
sb.append(" <messagesIn>").append(connectionStats.getReceivedMessages()).append("</messagesIn>\n");
|
||||
sb.append(" <messagesOut>").append(connectionStats.getSentMessages()).append("</messagesOut>\n");
|
||||
connectorDetail.put("statsOn", true);
|
||||
connectorDetail.put("connections", connectionStats.getConnectionsTotal());
|
||||
connectorDetail.put("connectionsOpen>", connectionStats.getConnections());
|
||||
connectorDetail.put("connectionsOpenMax", connectionStats.getConnectionsMax());
|
||||
connectorDetail.put("connectionsDurationMean", connectionStats.getConnectionDurationMean());
|
||||
connectorDetail.put("connectionsDurationMax", connectionStats.getConnectionDurationMax());
|
||||
connectorDetail.put("connectionsDurationStdDev", connectionStats.getConnectionDurationStdDev());
|
||||
connectorDetail.put("bytesIn", connectionStats.getReceivedBytes());
|
||||
connectorDetail.put("bytesOut", connectionStats.getSentBytes());
|
||||
connectorDetail.put("messagesIn", connectionStats.getReceivedMessages());
|
||||
connectorDetail.put("messagesOut", connectionStats.getSentMessages());
|
||||
}
|
||||
else
|
||||
{
|
||||
// Support for deprecated ConnectorStatistics (will be dropped in Jetty 10+)
|
||||
ConnectorStatistics connectorStats = null;
|
||||
if (connector instanceof AbstractConnector)
|
||||
connectorStats = connector.getBean(ConnectorStatistics.class);
|
||||
if (connectorStats != null)
|
||||
{
|
||||
sb.append(" <statsOn>true</statsOn>\n");
|
||||
sb.append(" <connections>").append(connectorStats.getConnections()).append("</connections>\n");
|
||||
sb.append(" <connectionsOpen>").append(connectorStats.getConnectionsOpen()).append("</connectionsOpen>\n");
|
||||
sb.append(" <connectionsOpenMax>").append(connectorStats.getConnectionsOpenMax()).append("</connectionsOpenMax>\n");
|
||||
sb.append(" <connectionsDurationMean>").append(connectorStats.getConnectionDurationMean()).append("</connectionsDurationMean>\n");
|
||||
sb.append(" <connectionsDurationMax>").append(connectorStats.getConnectionDurationMax()).append("</connectionsDurationMax>\n");
|
||||
sb.append(" <connectionsDurationStdDev>").append(connectorStats.getConnectionDurationStdDev()).append("</connectionsDurationStdDev>\n");
|
||||
sb.append(" <messagesIn>").append(connectorStats.getMessagesIn()).append("</messagesIn>\n");
|
||||
sb.append(" <messagesOut>").append(connectorStats.getMessagesIn()).append("</messagesOut>\n");
|
||||
sb.append(" <elapsedMs>").append(connectorStats.getStartedMillis()).append("</elapsedMs>\n");
|
||||
connectorDetail.put("statsOn", true);
|
||||
connectorDetail.put("connections", connectorStats.getConnections());
|
||||
connectorDetail.put("connectionsOpen", connectorStats.getConnectionsOpen());
|
||||
connectorDetail.put("connectionsOpenMax", connectorStats.getConnectionsOpenMax());
|
||||
connectorDetail.put("connectionsDurationMean", connectorStats.getConnectionDurationMean());
|
||||
connectorDetail.put("connectionsDurationMax", connectorStats.getConnectionDurationMax());
|
||||
connectorDetail.put("connectionsDurationStdDev", connectorStats.getConnectionDurationStdDev());
|
||||
connectorDetail.put("messagesIn", connectorStats.getMessagesIn());
|
||||
connectorDetail.put("messagesOut", connectorStats.getMessagesIn());
|
||||
connectorDetail.put("elapsedMs", connectorStats.getStartedMillis());
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.append(" <statsOn>false</statsOn>\n");
|
||||
connectorDetail.put("statsOn", false);
|
||||
}
|
||||
}
|
||||
sb.append(" </connector>\n");
|
||||
}
|
||||
sb.append(" </connections>\n");
|
||||
connections.add(connectorDetail);
|
||||
});
|
||||
top.put("connections", connections);
|
||||
|
||||
sb.append(" <memory>\n");
|
||||
sb.append(" <heapMemoryUsage>").append(_memoryBean.getHeapMemoryUsage().getUsed()).append("</heapMemoryUsage>\n");
|
||||
sb.append(" <nonHeapMemoryUsage>").append(_memoryBean.getNonHeapMemoryUsage().getUsed()).append("</nonHeapMemoryUsage>\n");
|
||||
sb.append(" </memory>\n");
|
||||
// memory
|
||||
Map<String, Number> memoryMap = new HashMap<>();
|
||||
memoryMap.put("heapMemoryUsage", _memoryBean.getHeapMemoryUsage().getUsed());
|
||||
memoryMap.put("nonHeapMemoryUsage", _memoryBean.getNonHeapMemoryUsage().getUsed());
|
||||
top.put("memory", memoryMap);
|
||||
|
||||
sb.append("</statistics>\n");
|
||||
|
||||
response.setContentType("text/xml");
|
||||
PrintWriter pout = response.getWriter();
|
||||
pout.write(sb.toString());
|
||||
// the top level object
|
||||
return outputProducer.generate("statistics", top);
|
||||
}
|
||||
|
||||
private void sendTextResponse(HttpServletResponse response) throws IOException
|
||||
private interface OutputProducer
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(_statsHandler.toStatsHTML());
|
||||
CharSequence generate(String id, Map<String, Object> map);
|
||||
}
|
||||
|
||||
sb.append("<h2>Connections:</h2>\n");
|
||||
for (Connector connector : _connectors)
|
||||
private static class JsonProducer implements OutputProducer
|
||||
{
|
||||
@Override
|
||||
public CharSequence generate(String id, Map<String, Object> map)
|
||||
{
|
||||
sb.append("<h3>").append(connector.getClass().getName()).append("@").append(connector.hashCode()).append("</h3>");
|
||||
sb.append("Protocols:");
|
||||
for (String protocol : connector.getProtocols())
|
||||
{
|
||||
sb.append(protocol).append(" ");
|
||||
}
|
||||
sb.append(" <br />\n");
|
||||
return JSON.toString(map);
|
||||
}
|
||||
}
|
||||
|
||||
ConnectionStatistics connectionStats = null;
|
||||
if (connector instanceof Container)
|
||||
connectionStats = connector.getBean(ConnectionStatistics.class);
|
||||
if (connectionStats != null)
|
||||
private static class XmlProducer implements OutputProducer
|
||||
{
|
||||
private final StringBuilder sb;
|
||||
private int indent = 0;
|
||||
|
||||
public XmlProducer()
|
||||
{
|
||||
this.sb = new StringBuilder();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharSequence generate(String id, Map<String, Object> map)
|
||||
{
|
||||
add(id, map);
|
||||
return sb;
|
||||
}
|
||||
|
||||
private void indent()
|
||||
{
|
||||
sb.append("\n");
|
||||
for (int i = 0; i < indent; i++)
|
||||
{
|
||||
sb.append("Total connections: ").append(connectionStats.getConnectionsTotal()).append("<br />\n");
|
||||
sb.append("Current connections open: ").append(connectionStats.getConnections()).append("<br />\n");
|
||||
sb.append("Max concurrent connections open: ").append(connectionStats.getConnectionsMax()).append("<br />\n");
|
||||
sb.append("Mean connection duration: ").append(connectionStats.getConnectionDurationMean()).append("<br />\n");
|
||||
sb.append("Max connection duration: ").append(connectionStats.getConnectionDurationMax()).append("<br />\n");
|
||||
sb.append("Connection duration standard deviation: ").append(connectionStats.getConnectionDurationStdDev()).append("<br />\n");
|
||||
sb.append("Total bytes received: ").append(connectionStats.getReceivedBytes()).append("<br />\n");
|
||||
sb.append("Total bytes sent: ").append(connectionStats.getSentBytes()).append("<br />\n");
|
||||
sb.append("Total messages received: ").append(connectionStats.getReceivedMessages()).append("<br />\n");
|
||||
sb.append("Total messages sent: ").append(connectionStats.getSentMessages()).append("<br />\n");
|
||||
}
|
||||
else
|
||||
{
|
||||
ConnectorStatistics connectorStats = null;
|
||||
if (connector instanceof AbstractConnector)
|
||||
connectorStats = connector.getBean(ConnectorStatistics.class);
|
||||
if (connectorStats != null)
|
||||
{
|
||||
sb.append("Statistics gathering started ").append(connectorStats.getStartedMillis()).append("ms ago").append("<br />\n");
|
||||
sb.append("Total connections: ").append(connectorStats.getConnections()).append("<br />\n");
|
||||
sb.append("Current connections open: ").append(connectorStats.getConnectionsOpen()).append("<br />\n");
|
||||
sb.append("Max concurrent connections open: ").append(connectorStats.getConnectionsOpenMax()).append("<br />\n");
|
||||
sb.append("Mean connection duration: ").append(connectorStats.getConnectionDurationMean()).append("<br />\n");
|
||||
sb.append("Max connection duration: ").append(connectorStats.getConnectionDurationMax()).append("<br />\n");
|
||||
sb.append("Connection duration standard deviation: ").append(connectorStats.getConnectionDurationStdDev()).append("<br />\n");
|
||||
sb.append("Total messages in: ").append(connectorStats.getMessagesIn()).append("<br />\n");
|
||||
sb.append("Total messages out: ").append(connectorStats.getMessagesOut()).append("<br />\n");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.append("Statistics gathering off.\n");
|
||||
}
|
||||
sb.append(' ').append(' ');
|
||||
}
|
||||
}
|
||||
|
||||
sb.append("<h2>Memory:</h2>\n");
|
||||
sb.append("Heap memory usage: ").append(_memoryBean.getHeapMemoryUsage().getUsed()).append(" bytes").append("<br />\n");
|
||||
sb.append("Non-heap memory usage: ").append(_memoryBean.getNonHeapMemoryUsage().getUsed()).append(" bytes").append("<br />\n");
|
||||
private void add(String id, Object obj)
|
||||
{
|
||||
sb.append('<').append(StringUtil.sanitizeXmlString(id)).append('>');
|
||||
indent++;
|
||||
|
||||
response.setContentType("text/html");
|
||||
PrintWriter pout = response.getWriter();
|
||||
pout.write(sb.toString());
|
||||
boolean wasIndented = false;
|
||||
|
||||
if (obj instanceof Map)
|
||||
{
|
||||
//noinspection unchecked
|
||||
addMap((Map<String, ?>)obj);
|
||||
wasIndented = true;
|
||||
}
|
||||
else if (obj instanceof List)
|
||||
{
|
||||
addList(id, (List<?>)obj);
|
||||
wasIndented = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
addObject(obj);
|
||||
}
|
||||
|
||||
indent--;
|
||||
if (wasIndented)
|
||||
indent();
|
||||
sb.append("</").append(id).append('>');
|
||||
}
|
||||
|
||||
private void addMap(Map<String, ?> map)
|
||||
{
|
||||
map.keySet().stream().sorted()
|
||||
.forEach((key) ->
|
||||
{
|
||||
indent();
|
||||
add(key, map.get(key));
|
||||
});
|
||||
}
|
||||
|
||||
private void addList(String parentId, List<?> list)
|
||||
{
|
||||
// drop the 's' at the end.
|
||||
String childName = parentId.replaceFirst("s$", "");
|
||||
list.forEach((entry) ->
|
||||
{
|
||||
indent();
|
||||
add(childName, entry);
|
||||
});
|
||||
}
|
||||
|
||||
private void addObject(Object obj)
|
||||
{
|
||||
sb.append(StringUtil.sanitizeXmlString(Objects.toString(obj)));
|
||||
}
|
||||
}
|
||||
|
||||
private static class TextProducer implements OutputProducer
|
||||
{
|
||||
private final StringBuilder sb;
|
||||
private int indent = 0;
|
||||
|
||||
public TextProducer()
|
||||
{
|
||||
this.sb = new StringBuilder();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharSequence generate(String id, Map<String, Object> map)
|
||||
{
|
||||
add(id, map);
|
||||
return sb;
|
||||
}
|
||||
|
||||
private void indent()
|
||||
{
|
||||
for (int i = 0; i < indent; i++)
|
||||
{
|
||||
sb.append(' ').append(' ');
|
||||
}
|
||||
}
|
||||
|
||||
private void add(String id, Object obj)
|
||||
{
|
||||
indent();
|
||||
sb.append(id).append(": ");
|
||||
indent++;
|
||||
|
||||
if (obj instanceof Map)
|
||||
{
|
||||
sb.append('\n');
|
||||
//noinspection unchecked
|
||||
addMap((Map<String, ?>)obj);
|
||||
}
|
||||
else if (obj instanceof List)
|
||||
{
|
||||
sb.append('\n');
|
||||
addList(id, (List<?>)obj);
|
||||
}
|
||||
else
|
||||
{
|
||||
addObject(obj);
|
||||
sb.append('\n');
|
||||
}
|
||||
|
||||
indent--;
|
||||
}
|
||||
|
||||
private void addMap(Map<String, ?> map)
|
||||
{
|
||||
map.keySet().stream().sorted()
|
||||
.forEach((key) -> add(key, map.get(key)));
|
||||
}
|
||||
|
||||
private void addList(String parentId, List<?> list)
|
||||
{
|
||||
// drop the 's' at the end.
|
||||
String childName = parentId.replaceFirst("s$", "");
|
||||
list.forEach((entry) -> add(childName, entry));
|
||||
}
|
||||
|
||||
private void addObject(Object obj)
|
||||
{
|
||||
sb.append(obj);
|
||||
}
|
||||
}
|
||||
|
||||
private static class HtmlProducer implements OutputProducer
|
||||
{
|
||||
private final StringBuilder sb;
|
||||
private int indent = 0;
|
||||
|
||||
public HtmlProducer()
|
||||
{
|
||||
this.sb = new StringBuilder();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharSequence generate(String id, Map<String, Object> map)
|
||||
{
|
||||
sb.append("<ul>\n");
|
||||
add(id, map);
|
||||
sb.append("</ul>\n");
|
||||
return sb;
|
||||
}
|
||||
|
||||
private void indent()
|
||||
{
|
||||
for (int i = 0; i < indent; i++)
|
||||
{
|
||||
sb.append(' ').append(' ');
|
||||
}
|
||||
}
|
||||
|
||||
private void add(String id, Object obj)
|
||||
{
|
||||
indent();
|
||||
indent++;
|
||||
sb.append("<li><em>").append(StringUtil.sanitizeXmlString(id)).append("</em>: ");
|
||||
if (obj instanceof Map)
|
||||
{
|
||||
//noinspection unchecked
|
||||
addMap((Map<String, ?>)obj);
|
||||
indent();
|
||||
}
|
||||
else if (obj instanceof List)
|
||||
{
|
||||
addList(id, (List<?>)obj);
|
||||
indent();
|
||||
}
|
||||
else
|
||||
{
|
||||
addObject(obj);
|
||||
}
|
||||
sb.append("</li>\n");
|
||||
|
||||
indent--;
|
||||
}
|
||||
|
||||
private void addMap(Map<String, ?> map)
|
||||
{
|
||||
sb.append("\n");
|
||||
indent();
|
||||
sb.append("<ul>\n");
|
||||
indent++;
|
||||
map.keySet().stream().sorted(String::compareToIgnoreCase)
|
||||
.forEach((key) -> add(key, map.get(key)));
|
||||
indent--;
|
||||
indent();
|
||||
sb.append("</ul>\n");
|
||||
}
|
||||
|
||||
private void addList(String parentId, List<?> list)
|
||||
{
|
||||
sb.append("\n");
|
||||
indent();
|
||||
sb.append("<ul>\n");
|
||||
indent++;
|
||||
// drop the 's' at the end.
|
||||
String childName = parentId.replaceFirst("s$", "");
|
||||
list.forEach((entry) -> add(childName, entry));
|
||||
indent--;
|
||||
indent();
|
||||
sb.append("</ul>\n");
|
||||
}
|
||||
|
||||
private void addObject(Object obj)
|
||||
{
|
||||
sb.append(StringUtil.sanitizeXmlString(Objects.toString(obj)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -18,31 +18,42 @@
|
|||
|
||||
package org.eclipse.jetty.servlet;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringReader;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Map;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.xpath.XPath;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
|
||||
import org.eclipse.jetty.http.HttpHeader;
|
||||
import org.eclipse.jetty.http.HttpTester;
|
||||
import org.eclipse.jetty.http.HttpVersion;
|
||||
import org.eclipse.jetty.server.LocalConnector;
|
||||
import org.eclipse.jetty.server.Server;
|
||||
import org.eclipse.jetty.server.handler.StatisticsHandler;
|
||||
import org.eclipse.jetty.server.session.SessionHandler;
|
||||
import org.eclipse.jetty.util.ajax.JSON;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.w3c.dom.Document;
|
||||
import org.xml.sax.InputSource;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
import static org.hamcrest.Matchers.not;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
public class StatisticsServletTest
|
||||
{
|
||||
|
@ -66,9 +77,7 @@ public class StatisticsServletTest
|
|||
_server.join();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getStats()
|
||||
throws Exception
|
||||
private void addStatisticsHandler()
|
||||
{
|
||||
StatisticsHandler statsHandler = new StatisticsHandler();
|
||||
_server.setHandler(statsHandler);
|
||||
|
@ -78,30 +87,196 @@ public class StatisticsServletTest
|
|||
servletHolder.setInitParameter("restrictToLocalhost", "false");
|
||||
statsContext.addServlet(servletHolder, "/stats");
|
||||
statsContext.setSessionHandler(new SessionHandler());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getStats()
|
||||
throws Exception
|
||||
{
|
||||
addStatisticsHandler();
|
||||
_server.start();
|
||||
|
||||
getResponse("/test1");
|
||||
String response = getResponse("/stats?xml=true");
|
||||
Stats stats = parseStats(response);
|
||||
HttpTester.Response response;
|
||||
|
||||
// Trigger 2xx response
|
||||
response = getResponse("/test1");
|
||||
assertEquals(response.getStatus(), 200);
|
||||
|
||||
// Look for 200 response that was tracked
|
||||
response = getResponse("/stats?xml=true");
|
||||
assertEquals(response.getStatus(), 200);
|
||||
Stats stats = parseStats(response.getContent());
|
||||
|
||||
assertEquals(1, stats.responses2xx);
|
||||
|
||||
getResponse("/stats?statsReset=true");
|
||||
// Reset stats
|
||||
response = getResponse("/stats?statsReset=true");
|
||||
assertEquals(response.getStatus(), 200);
|
||||
|
||||
// Request stats again
|
||||
response = getResponse("/stats?xml=true");
|
||||
stats = parseStats(response);
|
||||
assertEquals(response.getStatus(), 200);
|
||||
stats = parseStats(response.getContent());
|
||||
|
||||
assertEquals(1, stats.responses2xx);
|
||||
|
||||
getResponse("/test1");
|
||||
getResponse("/nothing");
|
||||
response = getResponse("/stats?xml=true");
|
||||
stats = parseStats(response);
|
||||
// Trigger 2xx response
|
||||
response = getResponse("/test1");
|
||||
assertEquals(response.getStatus(), 200);
|
||||
// Trigger 4xx response
|
||||
response = getResponse("/nothing");
|
||||
assertEquals(response.getStatus(), 404);
|
||||
|
||||
// Request stats again
|
||||
response = getResponse("/stats?xml=true");
|
||||
assertEquals(response.getStatus(), 200);
|
||||
stats = parseStats(response.getContent());
|
||||
|
||||
// Verify we see (from last reset)
|
||||
// 1) request for /stats?statsReset=true [2xx]
|
||||
// 2) request for /stats?xml=true [2xx]
|
||||
// 3) request for /test1 [2xx]
|
||||
// 4) request for /nothing [4xx]
|
||||
assertThat("2XX Response Count" + response, stats.responses2xx, is(3));
|
||||
assertThat("4XX Response Count" + response, stats.responses4xx, is(1));
|
||||
}
|
||||
|
||||
public String getResponse(String path)
|
||||
@Test
|
||||
public void getXmlResponse()
|
||||
throws Exception
|
||||
{
|
||||
addStatisticsHandler();
|
||||
_server.start();
|
||||
|
||||
HttpTester.Response response;
|
||||
HttpTester.Request request = new HttpTester.Request();
|
||||
|
||||
request.setMethod("GET");
|
||||
request.setURI("/stats");
|
||||
request.setHeader("Accept", "text/xml");
|
||||
request.setVersion(HttpVersion.HTTP_1_1);
|
||||
request.setHeader("Host", "test");
|
||||
|
||||
ByteBuffer responseBuffer = _connector.getResponse(request.generate());
|
||||
response = HttpTester.parseResponse(responseBuffer);
|
||||
|
||||
assertThat("Response.contentType", response.get(HttpHeader.CONTENT_TYPE), containsString("text/xml"));
|
||||
|
||||
// System.out.println(response.getContent());
|
||||
|
||||
// Parse it, make sure it's well formed.
|
||||
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
|
||||
try (ByteArrayInputStream input = new ByteArrayInputStream(response.getContentBytes()))
|
||||
{
|
||||
Document doc = docBuilder.parse(input);
|
||||
assertNotNull(doc);
|
||||
assertEquals("statistics", doc.getDocumentElement().getNodeName());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getJsonResponse()
|
||||
throws Exception
|
||||
{
|
||||
addStatisticsHandler();
|
||||
_server.start();
|
||||
|
||||
HttpTester.Response response;
|
||||
HttpTester.Request request = new HttpTester.Request();
|
||||
|
||||
request.setMethod("GET");
|
||||
request.setURI("/stats");
|
||||
request.setHeader("Accept", "application/json");
|
||||
request.setVersion(HttpVersion.HTTP_1_1);
|
||||
request.setHeader("Host", "test");
|
||||
|
||||
ByteBuffer responseBuffer = _connector.getResponse(request.generate());
|
||||
response = HttpTester.parseResponse(responseBuffer);
|
||||
|
||||
assertThat("Response.contentType", response.get(HttpHeader.CONTENT_TYPE), is("application/json"));
|
||||
assertThat("Response.contentType for json should never contain a charset",
|
||||
response.get(HttpHeader.CONTENT_TYPE), not(containsString("charset")));
|
||||
|
||||
// System.out.println(response.getContent());
|
||||
|
||||
// Parse it, make sure it's well formed.
|
||||
Object doc = JSON.parse(response.getContent());
|
||||
assertNotNull(doc);
|
||||
assertThat(doc, instanceOf(Map.class));
|
||||
Map<?, ?> docMap = (Map<?, ?>)doc;
|
||||
assertEquals(4, docMap.size());
|
||||
assertNotNull(docMap.get("requests"));
|
||||
assertNotNull(docMap.get("responses"));
|
||||
assertNotNull(docMap.get("connections"));
|
||||
assertNotNull(docMap.get("memory"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getTextResponse()
|
||||
throws Exception
|
||||
{
|
||||
addStatisticsHandler();
|
||||
_server.start();
|
||||
|
||||
HttpTester.Response response;
|
||||
HttpTester.Request request = new HttpTester.Request();
|
||||
|
||||
request.setMethod("GET");
|
||||
request.setURI("/stats");
|
||||
request.setHeader("Accept", "text/plain");
|
||||
request.setVersion(HttpVersion.HTTP_1_1);
|
||||
request.setHeader("Host", "test");
|
||||
|
||||
ByteBuffer responseBuffer = _connector.getResponse(request.generate());
|
||||
response = HttpTester.parseResponse(responseBuffer);
|
||||
|
||||
assertThat("Response.contentType", response.get(HttpHeader.CONTENT_TYPE), containsString("text/plain"));
|
||||
|
||||
// System.out.println(response.getContent());
|
||||
|
||||
// Look for expected content
|
||||
assertThat(response.getContent(), containsString("requests: "));
|
||||
assertThat(response.getContent(), containsString("responses: "));
|
||||
assertThat(response.getContent(), containsString("connections: "));
|
||||
assertThat(response.getContent(), containsString("memory: "));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getHtmlResponse()
|
||||
throws Exception
|
||||
{
|
||||
addStatisticsHandler();
|
||||
_server.start();
|
||||
|
||||
HttpTester.Response response;
|
||||
HttpTester.Request request = new HttpTester.Request();
|
||||
|
||||
request.setMethod("GET");
|
||||
request.setURI("/stats");
|
||||
request.setHeader("Accept", "text/html");
|
||||
request.setVersion(HttpVersion.HTTP_1_1);
|
||||
request.setHeader("Host", "test");
|
||||
|
||||
ByteBuffer responseBuffer = _connector.getResponse(request.generate());
|
||||
response = HttpTester.parseResponse(responseBuffer);
|
||||
|
||||
assertThat("Response.contentType", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html"));
|
||||
|
||||
System.out.println(response.getContent());
|
||||
|
||||
// Look for things that indicate it's a well formed HTML output
|
||||
assertThat(response.getContent(), containsString("<html>"));
|
||||
assertThat(response.getContent(), containsString("<body>"));
|
||||
assertThat(response.getContent(), containsString("<em>requests</em>: "));
|
||||
assertThat(response.getContent(), containsString("<em>responses</em>: "));
|
||||
assertThat(response.getContent(), containsString("<em>connections</em>: "));
|
||||
assertThat(response.getContent(), containsString("<em>memory</em>: "));
|
||||
assertThat(response.getContent(), containsString("</body>"));
|
||||
assertThat(response.getContent(), containsString("</html>"));
|
||||
}
|
||||
|
||||
public HttpTester.Response getResponse(String path)
|
||||
throws Exception
|
||||
{
|
||||
HttpTester.Request request = new HttpTester.Request();
|
||||
|
@ -111,7 +286,7 @@ public class StatisticsServletTest
|
|||
request.setHeader("Host", "test");
|
||||
|
||||
ByteBuffer responseBuffer = _connector.getResponse(request.generate());
|
||||
return HttpTester.parseResponse(responseBuffer).getContent();
|
||||
return HttpTester.parseResponse(responseBuffer);
|
||||
}
|
||||
|
||||
public Stats parseStats(String xml)
|
||||
|
@ -120,7 +295,6 @@ public class StatisticsServletTest
|
|||
XPath xPath = XPathFactory.newInstance().newXPath();
|
||||
|
||||
String responses4xx = xPath.evaluate("//responses4xx", new InputSource(new StringReader(xml)));
|
||||
|
||||
String responses2xx = xPath.evaluate("//responses2xx", new InputSource(new StringReader(xml)));
|
||||
|
||||
return new Stats(Integer.parseInt(responses2xx), Integer.parseInt(responses4xx));
|
||||
|
|
Loading…
Reference in New Issue