Fix for benchmark test
- Fix bug where repeatedly calling computeSummaryStatistics() could accumulate some values incorrectly. - Fix check for number of responsive nodes on list is <= number of candidate benchmark nodes. - Add public getters for summary statistics - Add javadoc for new getters - Add javadoc comments about API use
This commit is contained in:
Andrew Selden
parent
82aad78ff2
commit
c00120b818
|
|
@ -37,6 +37,14 @@ public class CompetitionDetails implements ToXContent {
|
|||
this.nodeResults = nodeResults;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets node-level competition results
|
||||
* @return A list of node-level competition results
|
||||
*/
|
||||
public List<CompetitionNodeResult> getNodeResults() {
|
||||
return nodeResults;
|
||||
}
|
||||
|
||||
@Override
|
||||
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
|
||||
|
|
@ -106,6 +114,18 @@ public class CompetitionDetails implements ToXContent {
|
|||
return builder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates detailed statistics for each iteration. Should be called prior to
|
||||
* accessing individual measurements.
|
||||
*/
|
||||
public void computeAllStatistics() {
|
||||
for (CompetitionNodeResult nodeResult : nodeResults) {
|
||||
for (CompetitionIteration iteration : nodeResult.iterations()) {
|
||||
iteration.computeStatistics();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private CompetitionIteration prototypicalIteration() {
|
||||
if (nodeResults != null && nodeResults.size() > 0) {
|
||||
CompetitionNodeResult nodeResult = nodeResults.get(0);
|
||||
|
|
@ -116,14 +136,6 @@ public class CompetitionDetails implements ToXContent {
|
|||
return null;
|
||||
}
|
||||
|
||||
private void computeAllStatistics() {
|
||||
for (CompetitionNodeResult nodeResult : nodeResults) {
|
||||
for (CompetitionIteration iteration : nodeResult.iterations()) {
|
||||
iteration.computeStatistics();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private int highestCompletedIteration() {
|
||||
int count = 0;
|
||||
for (CompetitionNodeResult nodeResult : nodeResults) {
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ public class CompetitionIteration implements Streamable {
|
|||
|
||||
public void computeStatistics() {
|
||||
|
||||
SinglePassStatistics single = new SinglePassStatistics();
|
||||
final SinglePassStatistics single = new SinglePassStatistics();
|
||||
|
||||
for (long datum : iterationData.data()) {
|
||||
if (datum > -1) { // ignore unset values in the underlying array
|
||||
|
|
|
|||
|
|
@ -34,29 +34,34 @@ import java.util.*;
|
|||
*
|
||||
* Statistics are calculated over all iteration results for all nodes
|
||||
* that executed the competition.
|
||||
*
|
||||
* Since values are calculated lazily on first access, users of this class
|
||||
* should first call computeSummaryStatistics() prior to accessing individual
|
||||
* measurements.
|
||||
*/
|
||||
public class CompetitionSummary implements ToXContent {
|
||||
|
||||
private List<CompetitionNodeResult> nodeResults;
|
||||
private boolean computed = false;
|
||||
|
||||
long min = 0;
|
||||
long max = 0;
|
||||
long totalTime = 0;
|
||||
long sumTotalHits = 0;
|
||||
long totalIterations = 0;
|
||||
long completedIterations = 0;
|
||||
long totalQueries = 0;
|
||||
double avgWarmupTime = 0;
|
||||
int concurrency = 0;
|
||||
int multiplier = 0;
|
||||
double mean = 0;
|
||||
double millisPerHit = 0.0;
|
||||
double stdDeviation = 0.0;
|
||||
double queriesPerSecond = 0.0;
|
||||
double[] percentiles;
|
||||
private long min = 0;
|
||||
private long max = 0;
|
||||
private long totalTime = 0;
|
||||
private long sumTotalHits = 0;
|
||||
private long totalIterations = 0;
|
||||
private long completedIterations = 0;
|
||||
private long totalQueries = 0;
|
||||
private double avgWarmupTime = 0;
|
||||
private int concurrency = 0;
|
||||
private int multiplier = 0;
|
||||
private double mean = 0;
|
||||
private double millisPerHit = 0.0;
|
||||
private double stdDeviation = 0.0;
|
||||
private double queriesPerSecond = 0.0;
|
||||
private double[] percentiles;
|
||||
Map<Double, Double> percentileValues = new TreeMap<>();
|
||||
|
||||
List<Tuple<String, CompetitionIteration.SlowRequest>> slowest = new ArrayList<>();
|
||||
private List<Tuple<String, CompetitionIteration.SlowRequest>> slowest = new ArrayList<>();
|
||||
|
||||
public CompetitionSummary() { }
|
||||
|
||||
|
|
@ -67,14 +72,26 @@ public class CompetitionSummary implements ToXContent {
|
|||
this.percentiles = percentiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets node-level competition results
|
||||
* @return A list of node-level competition results
|
||||
*/
|
||||
public List<CompetitionNodeResult> nodeResults() {
|
||||
return nodeResults;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates statistical measures from raw measurements. Should be called prior to accessing
|
||||
* individual measurements.
|
||||
*/
|
||||
public void computeSummaryStatistics() {
|
||||
|
||||
if (computed) {
|
||||
return;
|
||||
}
|
||||
|
||||
long totalWarmupTime = 0;
|
||||
SinglePassStatistics single = new SinglePassStatistics();
|
||||
final SinglePassStatistics single = new SinglePassStatistics();
|
||||
|
||||
for (CompetitionNodeResult nodeResult : nodeResults) {
|
||||
|
||||
|
|
@ -126,6 +143,7 @@ public class CompetitionSummary implements ToXContent {
|
|||
return Long.compare(o2.v2().maxTimeTaken(), o1.v2().maxTimeTaken());
|
||||
}
|
||||
});
|
||||
computed = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -179,6 +197,150 @@ public class CompetitionSummary implements ToXContent {
|
|||
return builder;
|
||||
}
|
||||
|
||||
/**
|
||||
* List of per-node competition results
|
||||
* @return Per-node competition results
|
||||
*/
|
||||
public List<CompetitionNodeResult> getNodeResults() {
|
||||
return nodeResults;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shortest execution time of any search
|
||||
* @return Shortest execution time of any search
|
||||
*/
|
||||
public long getMin() {
|
||||
return min;
|
||||
}
|
||||
|
||||
/**
|
||||
* Longest execution time of any search
|
||||
* @return Longest execution time of any search
|
||||
*/
|
||||
public long getMax() {
|
||||
return max;
|
||||
}
|
||||
|
||||
/**
|
||||
* Total execution time
|
||||
* @return Total execution time
|
||||
*/
|
||||
public long getTotalTime() {
|
||||
return totalTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Total hit count
|
||||
* @return Total hit count
|
||||
*/
|
||||
public long getSumTotalHits() {
|
||||
return sumTotalHits;
|
||||
}
|
||||
|
||||
/**
|
||||
* Number of requested iterations
|
||||
* @return Number of requested iterations
|
||||
*/
|
||||
public long getTotalIterations() {
|
||||
return totalIterations;
|
||||
}
|
||||
|
||||
/**
|
||||
* Number of iterations actually completed
|
||||
* @return Number of iterations actually completed
|
||||
*/
|
||||
public long getCompletedIterations() {
|
||||
return completedIterations;
|
||||
}
|
||||
|
||||
/**
|
||||
* Total number of queries actually executed
|
||||
* @return Number of queries actually executed
|
||||
*/
|
||||
public long getTotalQueries() {
|
||||
return totalQueries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mean average of warmup times across all nodes
|
||||
* @return Average warmup time
|
||||
*/
|
||||
public double getAvgWarmupTime() {
|
||||
return avgWarmupTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Number of concurrent searches
|
||||
* @return Concurrency
|
||||
*/
|
||||
public int getConcurrency() {
|
||||
return concurrency;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loop multiplier
|
||||
* @return Multiplier
|
||||
*/
|
||||
public int getMultiplier() {
|
||||
return multiplier;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mean average
|
||||
* @return Mean average
|
||||
*/
|
||||
public double getMean() {
|
||||
return mean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Total time considered as a percentage of total hits
|
||||
* @return Milliseconds-per-hit
|
||||
*/
|
||||
public double getMillisPerHit() {
|
||||
return millisPerHit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Standard deviation from the mean of all measurements
|
||||
* @return Standard deviation
|
||||
*/
|
||||
public double getStdDeviation() {
|
||||
return stdDeviation;
|
||||
}
|
||||
|
||||
/**
|
||||
* Measurement of the queries-per-second calculated as: numQueries * (1000.0 / totalTime)
|
||||
* @return Queries-per-second
|
||||
*/
|
||||
public double getQueriesPerSecond() {
|
||||
return queriesPerSecond;
|
||||
}
|
||||
|
||||
/**
|
||||
* The user-requested percentiles to measure
|
||||
* @return Array of percentiles to measure
|
||||
*/
|
||||
public double[] getPercentiles() {
|
||||
return percentiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* A map of percentiles and their measurements.
|
||||
* @return A map of entries of (percentile, measurement)
|
||||
*/
|
||||
public Map<Double, Double> getPercentileValues() {
|
||||
return percentileValues;
|
||||
}
|
||||
|
||||
/**
|
||||
* A list of the N slowest requests and the node that each executed on.
|
||||
* @return A list of pairs of (node, request)
|
||||
*/
|
||||
public List<Tuple<String, CompetitionIteration.SlowRequest>> getSlowest() {
|
||||
return slowest;
|
||||
}
|
||||
|
||||
static final class Fields {
|
||||
static final XContentBuilderString SUMMARY = new XContentBuilderString("summary");
|
||||
static final XContentBuilderString NODES = new XContentBuilderString("nodes");
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ public class BenchmarkIntegrationTest extends ElasticsearchIntegrationTest {
|
|||
|
||||
final BenchmarkRequest request =
|
||||
BenchmarkTestUtil.randomRequest(client(),indices, numExecutorNodes, competitionSettingsMap);
|
||||
logger.info("--> submitting benchmark - competitors [{}] iterations [{}]", request.competitors().size(),
|
||||
logger.info("--> Submitting benchmark - competitors [{}] iterations [{}]", request.competitors().size(),
|
||||
request.settings().iterations());
|
||||
final BenchmarkResponse response = client().bench(request).actionGet();
|
||||
|
||||
|
|
@ -100,7 +100,7 @@ public class BenchmarkIntegrationTest extends ElasticsearchIntegrationTest {
|
|||
|
||||
final BenchmarkRequest request =
|
||||
BenchmarkTestUtil.randomRequest(client(), indices, numExecutorNodes, competitionSettingsMap);
|
||||
logger.info("--> submitting benchmark - competitors [{}] iterations [{}]", request.competitors().size(),
|
||||
logger.info("--> Submitting benchmark - competitors [{}] iterations [{}]", request.competitors().size(),
|
||||
request.settings().iterations());
|
||||
|
||||
final CountDownLatch countdown = new CountDownLatch(1);
|
||||
|
|
@ -132,7 +132,7 @@ public class BenchmarkIntegrationTest extends ElasticsearchIntegrationTest {
|
|||
assertFalse(benchmarkResponse.hasErrors());
|
||||
|
||||
for (CompetitionResult result : benchmarkResponse.competitionResults().values()) {
|
||||
assertThat(result.nodeResults().size(), equalTo(numExecutorNodes));
|
||||
assertThat(result.nodeResults().size(), lessThanOrEqualTo(numExecutorNodes));
|
||||
validateCompetitionResult(result, competitionSettingsMap.get(result.competitionName()), false);
|
||||
}
|
||||
}
|
||||
|
|
@ -155,7 +155,7 @@ public class BenchmarkIntegrationTest extends ElasticsearchIntegrationTest {
|
|||
|
||||
final BenchmarkRequest request =
|
||||
BenchmarkTestUtil.randomRequest(client(), indices, numExecutorNodes, competitionSettingsMap);
|
||||
logger.info("--> submitting benchmark - competitors [{}] iterations [{}]", request.competitors().size(),
|
||||
logger.info("--> Submitting benchmark - competitors [{}] iterations [{}]", request.competitors().size(),
|
||||
request.settings().iterations());
|
||||
|
||||
final CountDownLatch countdown = new CountDownLatch(1);
|
||||
|
|
@ -179,7 +179,7 @@ public class BenchmarkIntegrationTest extends ElasticsearchIntegrationTest {
|
|||
Thread.sleep(1000);
|
||||
|
||||
final AbortBenchmarkResponse response = client().prepareAbortBench(BENCHMARK_NAME).execute().actionGet();
|
||||
assertThat(response.getNodeResponses().size(), equalTo(numExecutorNodes));
|
||||
assertThat(response.getNodeResponses().size(), lessThanOrEqualTo(numExecutorNodes));
|
||||
assertThat(response.getBenchmarkName(), equalTo(BENCHMARK_NAME));
|
||||
|
||||
for (AbortBenchmarkNodeResponse nodeResponse : response.getNodeResponses()) {
|
||||
|
|
@ -251,18 +251,18 @@ public class BenchmarkIntegrationTest extends ElasticsearchIntegrationTest {
|
|||
final CompetitionSummary summary = result.competitionSummary();
|
||||
summary.computeSummaryStatistics();
|
||||
assertThat(summary, notNullValue());
|
||||
assertThat(summary.min, greaterThanOrEqualTo(0L));
|
||||
assertThat(summary.max, greaterThanOrEqualTo(summary.min));
|
||||
assertThat(summary.mean, greaterThanOrEqualTo((double) summary.min));
|
||||
assertThat(summary.mean, lessThanOrEqualTo((double) summary.max));
|
||||
assertThat(summary.totalTime, greaterThanOrEqualTo(0L));
|
||||
assertThat(summary.queriesPerSecond, greaterThanOrEqualTo(0.0));
|
||||
assertThat(summary.millisPerHit, greaterThanOrEqualTo(0.0));
|
||||
assertThat(summary.avgWarmupTime, greaterThanOrEqualTo(0.0));
|
||||
assertThat(summary.getMin(), greaterThanOrEqualTo(0L));
|
||||
assertThat(summary.getMax(), greaterThanOrEqualTo(summary.getMin()));
|
||||
assertThat(summary.getMean(), greaterThanOrEqualTo((double) summary.getMin()));
|
||||
assertThat(summary.getMean(), lessThanOrEqualTo((double) summary.getMax()));
|
||||
assertThat(summary.getTotalTime(), greaterThanOrEqualTo(0L));
|
||||
assertThat(summary.getQueriesPerSecond(), greaterThanOrEqualTo(0.0));
|
||||
assertThat(summary.getMillisPerHit(), greaterThanOrEqualTo(0.0));
|
||||
assertThat(summary.getAvgWarmupTime(), greaterThanOrEqualTo(0.0));
|
||||
if (strict) {
|
||||
assertThat((int) summary.totalIterations, equalTo(requestedSettings.iterations() * summary.nodeResults().size()));
|
||||
assertThat((int) summary.completedIterations, equalTo(requestedSettings.iterations() * summary.nodeResults().size()));
|
||||
assertThat((int) summary.totalQueries, equalTo(requestedSettings.iterations() * requestedSettings.multiplier() *
|
||||
assertThat((int) summary.getTotalIterations(), equalTo(requestedSettings.iterations() * summary.nodeResults().size()));
|
||||
assertThat((int) summary.getCompletedIterations(), equalTo(requestedSettings.iterations() * summary.nodeResults().size()));
|
||||
assertThat((int) summary.getTotalQueries(), equalTo(requestedSettings.iterations() * requestedSettings.multiplier() *
|
||||
requestedSettings.searchRequests().size() * summary.nodeResults().size()));
|
||||
validatePercentiles(summary.percentileValues);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue