From 50bb0cd8fc4d15f16aa7d092ab71ff8b35308692 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 3 Feb 2010 21:40:16 +0000 Subject: [PATCH] 301089 Improve statistics available in StatisticsHandler and AbstractConnector git-svn-id: svn+ssh://dev.eclipse.org/svnroot/rt/org.eclipse.jetty/jetty/trunk@1236 7e9141cc-0065-0410-87d8-b60c137991c4 --- .../org/eclipse/jetty/server/ServerStats.java | 131 +++++++++ .../jetty/server/AbstractConnectorTest.java | 254 ++++++++++++++++++ .../org/eclipse/jetty/util/RunningStats.java | 74 +++++ .../org/eclipse/jetty/util/SimpleStats.java | 190 +++++++++++++ .../eclipse/jetty/util/RunningStatsTest.java | 50 ++++ 5 files changed, 699 insertions(+) create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/ServerStats.java create mode 100644 jetty-server/src/test/java/org/eclipse/jetty/server/AbstractConnectorTest.java create mode 100644 jetty-util/src/main/java/org/eclipse/jetty/util/RunningStats.java create mode 100644 jetty-util/src/main/java/org/eclipse/jetty/util/SimpleStats.java create mode 100644 jetty-util/src/test/java/org/eclipse/jetty/util/RunningStatsTest.java diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/ServerStats.java b/jetty-server/src/main/java/org/eclipse/jetty/server/ServerStats.java new file mode 100644 index 00000000000..8c2aeb6c555 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/ServerStats.java @@ -0,0 +1,131 @@ +// ======================================================================== +// Copyright (c) 2009-2009 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== + + +package org.eclipse.jetty.server; + +import org.eclipse.jetty.util.RunningStats; +import org.eclipse.jetty.util.SimpleStats; + + +/* ------------------------------------------------------------ */ +/** + * ServerStats + * + * Aggregates classes that computes statistic values + */ +public class ServerStats +{ + private ServerStats() {} + + /* ------------------------------------------------------------ */ + /** + * CounterStats + * + * Computes statistic values for counter variables + */ + public static class CounterStats + extends SimpleStats + { + /* ------------------------------------------------------------ */ + /** + * Construct the request statistics object + */ + public CounterStats() + { + super(true, true, false, true); // track current, total, and max + } + + /* ------------------------------------------------------------ */ + /** + * @see org.eclipse.jetty.util.SimpleStats#set(long) + */ + @Override + public void set(long value) + { + throw new UnsupportedOperationException(); + } + } + + /* ------------------------------------------------------------ */ + /** + * MeasuredStats + * + * Computes statistic values for measured variables + */ + public static class MeasuredStats + extends SimpleStats + { + private final RunningStats _stats = new RunningStats(); + + /* ------------------------------------------------------------ */ + /** + * Construct the request statistics object + */ + public MeasuredStats() + { + super(false, true, false, true); // track total and max only + } + + + /* ------------------------------------------------------------ */ + /** + * @see org.eclipse.jetty.util.SimpleStats#reset() + */ + @Override + public void reset() + { + super.reset(); + _stats.reset(); + } + + /* ------------------------------------------------------------ */ + /** + * @see org.eclipse.jetty.util.SimpleStats#set(long) + */ + @Override + public void set(long value) + { + super.set(value); + _stats.update(value); + } + + /* ------------------------------------------------------------ */ + /** + * @see org.eclipse.jetty.util.SimpleStats#add(long) + */ + @Override + public void add(long delta) + { + throw new UnsupportedOperationException(); + } + + /* ------------------------------------------------------------ */ + /** + * @return mean value of requests per connection + */ + public double getMean() + { + return _stats.getMean(); + } + + /* ------------------------------------------------------------ */ + /** + * @return standard deviation of requests per connection + */ + public double getStdDev() + { + return _stats.getStdDev(); + } + } +} diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/AbstractConnectorTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/AbstractConnectorTest.java new file mode 100644 index 00000000000..5fe8d82a8ad --- /dev/null +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/AbstractConnectorTest.java @@ -0,0 +1,254 @@ +// ======================================================================== +// Copyright (c) 2004-2009 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== + +package org.eclipse.jetty.server; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.net.Socket; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import junit.framework.TestCase; + +import org.eclipse.jetty.io.Connection; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.eclipse.jetty.server.handler.HandlerWrapper; +import org.eclipse.jetty.server.nio.SelectChannelConnector; +import org.eclipse.jetty.util.log.Log; + +public class AbstractConnectorTest extends TestCase +{ + private Server _server; + private Handler _handler; + private AbstractConnector _connector; + + private String _request = "GET / HTTP/1.1\r\n" + + "Host: localhost\r\n" + + "\r\n"; + private Socket[] _socket; + private PrintWriter[] _out; + private BufferedReader[] _in; + + private CyclicBarrier _connect; + private CountDownLatch _closed; + + @Override + protected void setUp() throws Exception + { + _connect = new CyclicBarrier(2); + + _server = new Server(); + + _connector = + new SelectChannelConnector() { + public void connectionClosed(Connection connection) + { + super.connectionClosed(connection); + _closed.countDown(); + } + + }; + _server.addConnector(_connector); + _connector.setStatsOn(true); + + HandlerWrapper wrapper = + new HandlerWrapper() { + public void handle(String path, Request request, HttpServletRequest httpRequest, HttpServletResponse httpResponse) throws IOException, ServletException + { + try + { + _connect.await(); + } + catch (Exception ex) + { + Log.debug(ex); + } + finally + { + super.handle(path, request, httpRequest, httpResponse); + } + } + }; + _server.setHandler(wrapper); + + _handler = + new AbstractHandler() { + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException + { + baseRequest.setHandled(true); + + PrintWriter out = response.getWriter(); + out.write("Server response\n"); + out.close(); + + response.setStatus(HttpServletResponse.SC_OK); + } + }; + wrapper.setHandler(_handler); + + _server.start(); + } + + @Override + protected void tearDown() throws Exception + { + _server.stop(); + _server.join(); + } + + public void testSingleRequest() + throws Exception + { + doInit(1); + + sendRequest(1, 1); + + doClose(1); + + assertEquals(1, _connector.getConnections()); + assertEquals(0, _connector.getConnectionsOpen()); + assertEquals(1, _connector.getConnectionsOpenMax()); + assertTrue(_connector.getConnectionsOpen() <= _connector.getConnectionsOpenMax()); + + assertTrue(_connector.getConnectionsDurationMean() > 0); + assertTrue(_connector.getConnectionsDurationMax() > 0); + assertTrue(_connector.getConnectionsDurationMean() <= _connector.getConnectionsDurationMax()); + + assertEquals(1, _connector.getRequests()); + assertEquals(1.0, _connector.getConnectionsRequestsMean()); + assertEquals(1, _connector.getConnectionsRequestsMax()); + assertTrue(_connector.getConnectionsRequestsMean() <= _connector.getConnectionsRequestsMax()); + } + + public void testMultipleRequests() + throws Exception + { + doInit(1); + + sendRequest(1, 1); + + sendRequest(1, 1); + + doClose(1); + + assertEquals(1, _connector.getConnections()); + assertEquals(0, _connector.getConnectionsOpen()); + assertEquals(1, _connector.getConnectionsOpenMax()); + assertTrue(_connector.getConnectionsOpen() <= _connector.getConnectionsOpenMax()); + + assertTrue(_connector.getConnectionsDurationMean() > 0); + assertTrue(_connector.getConnectionsDurationMax() > 0); + assertTrue(_connector.getConnectionsDurationMean() <= _connector.getConnectionsDurationMax()); + + assertEquals(2, _connector.getRequests()); + assertEquals(2.0, _connector.getConnectionsRequestsMean()); + assertEquals(2, _connector.getConnectionsRequestsMax()); + assertTrue(_connector.getConnectionsRequestsMean() <= _connector.getConnectionsRequestsMax()); + } + + public void testMultipleConnections() + throws Exception + { + doInit(3); + + sendRequest(1, 1); // request 1 connection 1 + + sendRequest(2, 2); // request 1 connection 2 + + sendRequest(3, 3); // request 1 connection 3 + + sendRequest(2, 3); // request 2 connection 2 + + sendRequest(3, 3); // request 2 connection 3 + + sendRequest(3, 3); // request 3 connection 3 + + doClose(3); + + assertEquals(3, _connector.getConnections()); + assertEquals(0, _connector.getConnectionsOpen()); + assertEquals(3, _connector.getConnectionsOpenMax()); + assertTrue(_connector.getConnectionsOpen() <= _connector.getConnectionsOpenMax()); + + assertTrue(_connector.getConnectionsDurationMean() > 0); + assertTrue(_connector.getConnectionsDurationMax() > 0); + assertTrue(_connector.getConnectionsDurationMean() <= _connector.getConnectionsDurationMax()); + + assertEquals(6, _connector.getRequests()); + assertEquals(2.0, _connector.getConnectionsRequestsMean()); + assertEquals(3, _connector.getConnectionsRequestsMax()); + assertTrue(_connector.getConnectionsRequestsMean() <= _connector.getConnectionsRequestsMax()); + } + + protected void doInit(int count) + { + _socket = new Socket[count]; + _out = new PrintWriter[count]; + _in = new BufferedReader[count]; + + _closed = new CountDownLatch(count); + } + + protected void doClose(int count) + throws Exception + { + for (int idx=0; idx < count; idx++) + { + if (_out[idx] != null) + _out[idx].close(); + + if (_in[idx] != null) + _in[idx].close(); + + if (_socket[idx] != null) + _socket[idx].close(); + } + + _closed.await(); + } + + protected void sendRequest(int id, int count) + throws Exception + { + int idx = id - 1; + + if (idx < 0) + throw new IllegalArgumentException("Connection ID <= 0"); + + _socket[idx] = _socket[idx] == null ? new Socket("localhost", _connector.getLocalPort()) : _socket[idx]; + _out[idx] = _out[idx] == null ? new PrintWriter(_socket[idx].getOutputStream(), true) : _out[idx]; + _in[idx] = _in[idx] == null ? new BufferedReader(new InputStreamReader(_socket[idx].getInputStream())) : _in[idx]; + + _connect.reset(); + + _out[idx].write(_request); + _out[idx].flush(); + + _connect.await(); + + assertEquals(count, _connector.getConnectionsOpen()); + + while(_in[idx].ready()) + { + _in[idx].readLine(); + } + } +} diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/RunningStats.java b/jetty-util/src/main/java/org/eclipse/jetty/util/RunningStats.java new file mode 100644 index 00000000000..c6c4def1332 --- /dev/null +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/RunningStats.java @@ -0,0 +1,74 @@ +// ======================================================================== +// Copyright (c) 2009-2009 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== + +package org.eclipse.jetty.util; + + +/* ------------------------------------------------------------ */ +/** + * StatsEstimator + * + * Calculates estimates of mean, variance, and standard deviation + * characteristics of a sample using on-line algorithm presented + * in Donald Knuth's Art of Computer Programming, Volume 2, + * Seminumerical Algorithms, 3rd edition, page 232, + * Boston: Addison-Wesley. that cites a 1962 paper by B.P. Welford + * that can be found by following the link below. + * + * http://www.jstor.org/pss/1266577 + * + * This algorithm can be found in Wikipedia article about computing + * standard deviation found by following the link below. + * + * http://en.wikipedia.org/w/index.php?title=Algorithms_for_calculating_variance§ion=4#On-line_algorithm + */ +public class RunningStats +{ + private volatile long _size; + private volatile double _mean; + private volatile double _rsum; + + public synchronized void reset() + { + _size = 0; + _mean = 0.0; + _rsum = 0.0; + } + + public synchronized void update(final double x) + { + double mean = _mean; + _mean += (x - mean) / ++_size; + _rsum += (x - mean) * (x - _mean); + } + + public long getSize() + { + return _size; + } + + public double getMean() + { + return _mean; + } + + public synchronized double getVariance() + { + return _size > 1 ? _rsum/(_size-1) : 0.0; + } + + public synchronized double getStdDev() + { + return Math.sqrt(getVariance()); + } +} diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/SimpleStats.java b/jetty-util/src/main/java/org/eclipse/jetty/util/SimpleStats.java new file mode 100644 index 00000000000..e08c8b3e956 --- /dev/null +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/SimpleStats.java @@ -0,0 +1,190 @@ +// ======================================================================== +// Copyright (c) 2009-2009 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== + +package org.eclipse.jetty.util; + +import static java.lang.Math.abs; + +import java.util.concurrent.atomic.AtomicLong; + + +/* ------------------------------------------------------------ */ +/** + */ +public class SimpleStats +{ + private final AtomicLong _curr; + private final AtomicLong _total; + private final AtomicLong _min; + private final AtomicLong _max; + + /* ------------------------------------------------------------ */ + /** + */ + public SimpleStats() + { + this(true, true, true, true); + } + + /* ------------------------------------------------------------ */ + /** + * @param curr + * @param total + * @param min + * @param max + */ + public SimpleStats(boolean curr, boolean total, boolean min, boolean max) + { + _curr = new AtomicLong(curr ? 0 : -1); + _total = new AtomicLong(total ? 0 : -1); + _min = new AtomicLong(min ? 0 : -1); + _max = new AtomicLong(max ? 0 : -1); + } + + public void reset() + { + if (_curr.get() != -1) + _curr.set(0); + + if (_total.get() != -1) + _total.set(0); + + if (_min.get() != -1) + _min.set(0); + + if (_max.get() != -1) + _max.set(0); + } + + /* ------------------------------------------------------------ */ + /** + * @param value + */ + public void set(final long value) + { + if (_curr.get() != -1) + _curr.set(value); + + if (_total.get() != -1) + _total.addAndGet(value); + + if (_min.get() != -1) + updateMin(value); + + if (_max.get() != -1) + updateMax(value); + } + + /* ------------------------------------------------------------ */ + /** + * @param delta + */ + public void add(final long delta) + { + if (_curr.get() != -1) + { + long curr = _curr.addAndGet(delta); + + if (_min.get() != -1) + updateMin(curr); + + if (_max.get() != -1) + updateMax(curr); + } + + if (_total.get() != -1 && delta < 0) + _total.addAndGet(abs(delta)); + } + + /* ------------------------------------------------------------ */ + /** + */ + public void increment() + { + add(1); + } + + /* ------------------------------------------------------------ */ + /** + */ + public void decrement() + { + add(-1); + } + + /* ------------------------------------------------------------ */ + /** + * @return + */ + public long getCurrent() + { + return _curr.get(); + } + + /* ------------------------------------------------------------ */ + /** + * @return + */ + public long getTotal() + { + return _total.get(); + } + + /* ------------------------------------------------------------ */ + /** + * @return + */ + public long getMax() + { + return _max.get(); + } + + /* ------------------------------------------------------------ */ + /** + * @return + */ + public long getMin() + { + return _min.get(); + } + + /* ------------------------------------------------------------ */ + /** + * @param value + */ + private void updateMin(long value) + { + long oldValue = _min.get(); + while (value < oldValue) + { + if (_min.compareAndSet(oldValue, value)) + break; + oldValue = _min.get(); + } + } + + /* ------------------------------------------------------------ */ + /** + * @param value + */ + private void updateMax(long value) + { + long oldValue = _max.get(); + while (value > oldValue) + { + if (_max.compareAndSet(oldValue, value)) + break; + oldValue = _max.get(); + } + } +} diff --git a/jetty-util/src/test/java/org/eclipse/jetty/util/RunningStatsTest.java b/jetty-util/src/test/java/org/eclipse/jetty/util/RunningStatsTest.java new file mode 100644 index 00000000000..e1ae9f84bd7 --- /dev/null +++ b/jetty-util/src/test/java/org/eclipse/jetty/util/RunningStatsTest.java @@ -0,0 +1,50 @@ +package org.eclipse.jetty.util; + +import junit.framework.TestCase; + + +/* ------------------------------------------------------------ */ +/** + * Test data and results used in this test are from National Institute + * of Standards and Technology Information Technology Laboratory study. + * + * http://www.itl.nist.gov/div898/handbook/eda/section3/eda359.htm + */ +public class RunningStatsTest extends TestCase +{ + private static double[] data1 = {608.781,689.556,618.134,680.203,726.232,518.655,740.447,666.830,710.272,751.669,697.979,708.583,624.972,695.070,769.391,720.186,723.657,703.700,697.626,714.980,657.712,609.989,650.771,707.977,712.199,709.631,703.160,744.822,719.217,619.137,753.333,677.933,735.919,695.274,504.167,693.333,625.000,596.667,640.898,720.506,700.748,691.604,636.738,731.667,635.079,716.926,759.581,673.903,736.648,675.957,729.230,697.239,728.499,797.662,668.530,815.754,777.392,712.140,663.622,684.181,629.012,640.193,644.156,642.469,639.090,439.418,614.664,537.161,656.773,659.534,695.278,734.040,687.665,710.858,701.716,382.133,719.744,756.820,690.978,670.864,670.308,660.062,790.382,714.750,716.959,603.363,713.796,444.963,723.276,745.527,778.333,723.349,708.229,681.667,566.085,687.448,597.500,637.410,755.864,692.945,766.532,725.663,698.818,760.000,775.272,708.885,727.201,642.560,690.773,688.333,743.973,682.461,761.430,691.542,643.392,697.075,708.229,746.467,744.819,655.029,715.224,614.417,761.363,716.106,659.502,730.781,546.928,734.203,682.051,701.341,759.729,689.942,769.424,715.286,776.197,547.099,619.942,696.046,573.109,638.794,708.193,502.825,632.633,683.382,684.812,738.161,671.492,709.771,685.199,624.973,757.363,633.417,658.754,664.666,663.009,773.226,708.261,739.086,667.786,674.481,695.688,588.288,545.610,752.305,684.523,717.159,721.343,750.623,776.488,750.623,600.840,686.196,687.870,725.527,658.796,690.380,737.144,663.851,766.630,625.922,694.430,730.217,700.770,722.242,763.828,695.668,688.887,531.021,698.915,735.905,732.039,751.832,618.663,744.845,690.826,666.893,759.860,683.752,729.591,730.706,763.124,724.193,630.352,750.338,752.417,707.899,715.582,728.746,591.193,592.252,740.833,786.367,712.386,738.333,741.480,729.167,795.833,723.502,718.333,768.080,747.500,775.000,760.599,758.333,682.500,658.116,738.213,681.236,704.904,693.623,624.993,700.228,611.874,579.167,720.872,690.320,677.933,674.600,611.999,530.680}; + private static double[] data2 = {569.670,747.541,612.182,607.766,605.380,589.226,588.375,531.384,633.417,619.060,632.447,624.256,575.143,549.278,624.972,587.695,569.207,613.257,565.737,662.131,543.177,512.394,611.190,659.982,569.245,725.792,608.960,586.060,617.441,592.845,631.754,588.113,555.724,702.411,631.754,698.254,616.791,551.953,636.738,571.551,521.667,587.451,700.422,595.819,534.236,606.188,575.303,590.628,729.314,619.313,624.234,651.304,724.175,583.034,620.227,584.861,565.391,622.506,628.336,587.145,584.319,538.239,538.097,595.686,648.935,583.827,534.905,569.858,617.246,610.337,584.192,598.853,554.774,605.694,627.516,574.522,582.682,563.872,715.962,616.430,778.011,604.255,571.906,625.925,682.426,707.604,617.400,689.576,676.678,563.290,581.879,447.701,557.772,593.537,632.585,671.350,569.530,581.667,643.449,581.593,494.122,620.948,615.903,606.667,579.167,662.510,436.237,644.223,586.035,620.833,652.535,593.516,587.451,570.964,645.192,540.079,707.117,621.779,585.777,703.980,698.237,757.120,621.751,472.125,612.700,583.170,599.771,549.227,605.453,569.599,637.233,621.774,558.041,583.170,345.294,570.999,603.232,595.335,581.047,455.878,627.880,464.085,596.129,640.371,621.471,612.727,606.460,571.760,599.304,579.459,761.511,566.969,654.397,611.719,577.409,576.731,617.441,577.409,548.957,623.315,621.761,553.978,657.157,610.882,552.304,545.303,651.934,635.240,641.083,645.321,566.127,647.844,554.815,620.087,711.301,644.355,713.812,696.707,589.453,634.468,599.751,624.542,723.505,674.717,608.539,612.135,591.935,676.656,647.323,811.970,603.883,608.643,630.778,623.063,472.463,645.932,577.176,567.530,821.654,684.490,600.427,686.023,628.109,605.214,640.260,700.767,665.924,555.926,543.299,511.030,583.994,611.048,623.338,679.585,665.004,655.860,715.711,611.999,577.722,615.129,540.316,711.667,639.167,549.491,684.167,672.153,594.534,627.650,551.870,594.534,602.660,585.450,555.724,574.934,584.625,555.724,611.874,698.254,748.130,689.942}; + + public void testData1() + throws Exception + { + RunningStats stats = new RunningStats(); + + for (double x : data1) + stats.update(x); + + assertEquals(data1.length, stats.getSize()); + assertEquals(688.9986, round(stats.getMean(), 4)); + assertEquals(4296.683, round(stats.getVariance(), 3)); + assertEquals(65.54909, round(stats.getStdDev(), 5)); + } + + public void testData2() + throws Exception + { + RunningStats stats = new RunningStats(); + + for (double x : data2) + stats.update(x); + + assertEquals(data2.length, stats.getSize()); + assertEquals(611.1560, round(stats.getMean(), 4)); + assertEquals(3825.948, round(stats.getVariance(), 3)); + assertEquals(61.85425, round(stats.getStdDev(), 5)); + } + + private double round(double x, int places) + { + return Math.round(x * Math.pow(10.0, (double) places)) / Math.pow(10.0, (double) places); + } +}