feat(benchpress): show more metrics and make the run mode configurable
Shows the metrics: script, render, gcAmount, gcAmountInScript, gcTime Run modes: - detect: auto detect whether to force gc - forceGc: forces a gc before every run and ignores no runs - noGcInScript: ignore runs that have gc while a script was executing - plain: does not force gc nor ignore runs Closes #368
This commit is contained in:
parent
82b1601a31
commit
77aa3ed61b
|
@ -29,14 +29,9 @@ function runClickBenchmark(config) {
|
|||
var buttons = config.buttons.map(function(selector) {
|
||||
return $(selector);
|
||||
});
|
||||
var timeParams = browser.params.benchmark;
|
||||
benchpress.runBenchmark({
|
||||
sampleSize: timeParams.sampleSize,
|
||||
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
|
||||
timeout: timeParams.timeout,
|
||||
metrics: timeParams.metrics,
|
||||
logId: browser.params.lang+'.'+config.logId
|
||||
}, function() {
|
||||
var params = Object.create(browser.params.benchmark);
|
||||
params.logId = browser.params.lang+'.'+config.logId;
|
||||
benchpress.runBenchmark(params, function() {
|
||||
buttons.forEach(function(button) {
|
||||
button.click();
|
||||
});
|
||||
|
|
|
@ -29,14 +29,9 @@ function runClickBenchmark(config) {
|
|||
var buttons = config.buttons.map(function(selector) {
|
||||
return $(selector);
|
||||
});
|
||||
var timeParams = browser.params.benchmark;
|
||||
benchpress.runBenchmark({
|
||||
sampleSize: timeParams.sampleSize,
|
||||
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
|
||||
timeout: timeParams.timeout,
|
||||
metrics: timeParams.metrics,
|
||||
logId: browser.params.lang+'.'+config.logId
|
||||
}, function() {
|
||||
var params = Object.create(browser.params.benchmark);
|
||||
params.logId = browser.params.lang+'.'+config.logId;
|
||||
benchpress.runBenchmark(params, function() {
|
||||
buttons.forEach(function(button) {
|
||||
button.click();
|
||||
});
|
||||
|
|
|
@ -45,14 +45,9 @@ function runClickBenchmark(config) {
|
|||
var buttons = config.buttons.map(function(selector) {
|
||||
return $(selector);
|
||||
});
|
||||
var timeParams = browser.params.benchmark;
|
||||
benchpress.runBenchmark({
|
||||
sampleSize: timeParams.sampleSize,
|
||||
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
|
||||
timeout: timeParams.timeout,
|
||||
metrics: timeParams.metrics,
|
||||
logId: browser.params.lang+'.'+config.logId
|
||||
}, function() {
|
||||
var params = Object.create(browser.params.benchmark);
|
||||
params.logId = browser.params.lang+'.'+config.logId;
|
||||
benchpress.runBenchmark(params, function() {
|
||||
buttons.forEach(function(button) {
|
||||
button.click();
|
||||
});
|
||||
|
|
|
@ -29,14 +29,9 @@ function runClickBenchmark(config) {
|
|||
var buttons = config.buttons.map(function(selector) {
|
||||
return $(selector);
|
||||
});
|
||||
var timeParams = browser.params.benchmark;
|
||||
benchpress.runBenchmark({
|
||||
sampleSize: timeParams.sampleSize,
|
||||
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
|
||||
timeout: timeParams.timeout,
|
||||
metrics: timeParams.metrics,
|
||||
logId: browser.params.lang+'.'+config.logId
|
||||
}, function() {
|
||||
var params = Object.create(browser.params.benchmark);
|
||||
params.logId = browser.params.lang+'.'+config.logId;
|
||||
benchpress.runBenchmark(params, function() {
|
||||
buttons.forEach(function(button) {
|
||||
button.click();
|
||||
});
|
||||
|
|
|
@ -29,14 +29,9 @@ function runClickBenchmark(config) {
|
|||
var buttons = config.buttons.map(function(selector) {
|
||||
return $(selector);
|
||||
});
|
||||
var timeParams = browser.params.benchmark;
|
||||
benchpress.runBenchmark({
|
||||
sampleSize: timeParams.sampleSize,
|
||||
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
|
||||
timeout: timeParams.timeout,
|
||||
metrics: timeParams.metrics,
|
||||
logId: browser.params.lang+'.'+config.logId
|
||||
}, function() {
|
||||
var params = Object.create(browser.params.benchmark);
|
||||
params.logId = browser.params.lang+'.'+config.logId;
|
||||
benchpress.runBenchmark(params, function() {
|
||||
buttons.forEach(function(button) {
|
||||
button.click();
|
||||
});
|
||||
|
|
|
@ -29,14 +29,9 @@ function runClickBenchmark(config) {
|
|||
var buttons = config.buttons.map(function(selector) {
|
||||
return $(selector);
|
||||
});
|
||||
var timeParams = browser.params.benchmark;
|
||||
benchpress.runBenchmark({
|
||||
sampleSize: timeParams.sampleSize,
|
||||
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
|
||||
timeout: timeParams.timeout,
|
||||
metrics: timeParams.metrics,
|
||||
logId: browser.params.lang+'.'+config.logId
|
||||
}, function() {
|
||||
var params = Object.create(browser.params.benchmark);
|
||||
params.logId = browser.params.lang+'.'+config.logId;
|
||||
benchpress.runBenchmark(params, function() {
|
||||
buttons.forEach(function(button) {
|
||||
button.click();
|
||||
});
|
||||
|
|
|
@ -21,14 +21,9 @@ function runClickBenchmark(config) {
|
|||
var buttons = config.buttons.map(function(selector) {
|
||||
return $(selector);
|
||||
});
|
||||
var timeParams = browser.params.benchmark;
|
||||
benchpress.runBenchmark({
|
||||
sampleSize: timeParams.sampleSize,
|
||||
targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
|
||||
timeout: timeParams.timeout,
|
||||
metrics: timeParams.metrics,
|
||||
logId: browser.params.lang+'.'+config.logId
|
||||
}, function() {
|
||||
var params = Object.create(browser.params.benchmark);
|
||||
params.logId = browser.params.lang+'.'+config.logId;
|
||||
benchpress.runBenchmark(params, function() {
|
||||
buttons.forEach(function(button) {
|
||||
button.click();
|
||||
});
|
||||
|
|
|
@ -11,7 +11,13 @@ var config = exports.config = {
|
|||
sampleSize: 10,
|
||||
targetCoefficientOfVariation: 4,
|
||||
timeout: 20000,
|
||||
metrics: ['script', 'render']
|
||||
metrics: ['script', 'render', 'gcAmount', 'gcAmountInScript', 'gcTime'],
|
||||
// run mode of the benchmark:
|
||||
// - detect: auto detect whether to force gc
|
||||
// - forceGc: forces a gc before every run and ignores no runs
|
||||
// - noGcInScript: ignore runs that have gc while a script was executing
|
||||
// - plain: does not force nor ignore runs
|
||||
mode: 'detect'
|
||||
}
|
||||
},
|
||||
|
||||
|
|
|
@ -6,16 +6,57 @@ var SUPPORTED_METRICS = {
|
|||
script: true,
|
||||
gcTime: true,
|
||||
gcAmount: true,
|
||||
gcTimeDuringScript: true,
|
||||
gcAmountDuringScript: true,
|
||||
gcTimeInScript: true,
|
||||
gcAmountInScript: true,
|
||||
gcAmountPerMs: true,
|
||||
render: true
|
||||
};
|
||||
var DETERMINE_FORCE_GC_MODE_ITERATIONS = 5;
|
||||
|
||||
var MODE_FORCE_GC = 'forceGc';
|
||||
var MODE_IGNORE_RUNS_WITH_GC = 'ignoreRunsWithGc';
|
||||
var MODE_INDETERMINATE = 'indeterminate';
|
||||
var RUN_MODE = {
|
||||
detect: function(prevState, benchmarkData, iterationIndex) {
|
||||
var gcInScriptCount = prevState.gcInScriptCount || 0;
|
||||
if (benchmarkData.gcAmountInScript) {
|
||||
gcInScriptCount++;
|
||||
}
|
||||
var ignoreRun = !!benchmarkData.gcAmountInScript;
|
||||
var nextMode = RUN_MODE.detect;
|
||||
if (iterationIndex > 10) {
|
||||
if (gcInScriptCount / iterationIndex > 0.7) {
|
||||
nextMode = RUN_MODE.forceGc;
|
||||
} else {
|
||||
nextMode = RUN_MODE.noGcInScript;
|
||||
}
|
||||
}
|
||||
return {
|
||||
forceGc: false,
|
||||
ignoreRun: ignoreRun,
|
||||
gcInScriptCount: gcInScriptCount,
|
||||
nextMode: nextMode
|
||||
};
|
||||
},
|
||||
forceGc: function() {
|
||||
return {
|
||||
forceGc: true,
|
||||
ignoreRun: false,
|
||||
nextMode: RUN_MODE.forceGc
|
||||
}
|
||||
},
|
||||
noGcInScript: function(prevState, benchmarkData) {
|
||||
var ignoreRun = !!benchmarkData.gcAmountInScript;
|
||||
return {
|
||||
forceGc: false,
|
||||
ignoreRun: ignoreRun,
|
||||
nextMode: RUN_MODE.noGcInScript
|
||||
}
|
||||
},
|
||||
plain: function() {
|
||||
return {
|
||||
forceGc: false,
|
||||
ignoreRun: false,
|
||||
nextMode: RUN_MODE.plain
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var nextTimestampId = 0;
|
||||
|
||||
|
@ -30,25 +71,30 @@ function runBenchmark(config, workCallback) {
|
|||
throw new Error('Metric '+metric+' is not suported by benchpress right now');
|
||||
}
|
||||
});
|
||||
var ROW_FORMAT = ['%-40s'].concat(config.metrics.map(function() {
|
||||
var ROW_FORMAT = ['%-40s', '%12s'].concat(config.metrics.map(function() {
|
||||
return '%12s';
|
||||
})).join(' | ');
|
||||
|
||||
var benchmarkStatsAggregator = stats.createObjectStatsAggregator(config.sampleSize);
|
||||
var benchmarkStatsAggregator = stats.createObjectStatsAggregator(config.metrics, config.sampleSize);
|
||||
|
||||
var startTime = Date.now();
|
||||
startLoop().then(endLoop);
|
||||
|
||||
var gcDuringScriptCount = 0;
|
||||
|
||||
function startLoop(gcData) {
|
||||
reporter.printHeading('SCRIPT DATA: sampling size '+config.sampleSize);
|
||||
reporter.printTableHeader(ROW_FORMAT, ['name'].concat(config.metrics));
|
||||
return loop(0, MODE_INDETERMINATE);
|
||||
reporter.printTableHeader(ROW_FORMAT, ['name', 'action'].concat(config.metrics));
|
||||
if (!(config.mode in RUN_MODE)) {
|
||||
throw new Error('Unknown mode '+config.mode);
|
||||
}
|
||||
return loop(0, {
|
||||
forceGc: false,
|
||||
ignoreRun: false,
|
||||
nextMode: RUN_MODE[config.mode]
|
||||
});
|
||||
}
|
||||
|
||||
function endLoop(stats) {
|
||||
reporter.printTableFooter(ROW_FORMAT, [config.logId]
|
||||
reporter.printTableFooter(ROW_FORMAT, [config.logId, '']
|
||||
.concat(formatObjectStats(stats, config.metrics))
|
||||
);
|
||||
return config.metrics.map(function(metric) {
|
||||
|
@ -56,40 +102,35 @@ function runBenchmark(config, workCallback) {
|
|||
});
|
||||
}
|
||||
|
||||
function loop(iterationIndex, mode) {
|
||||
function loop(iterationIndex, modeState) {
|
||||
return measureTime(function() {
|
||||
workCallback();
|
||||
if (modeState.forceGc) {
|
||||
// For fast tests that don't create a lot of garbage,
|
||||
// we don't want to force gc before every run as that
|
||||
// can slow down the script execution time (even when we subtract
|
||||
// the gc time)!
|
||||
if (mode === MODE_FORCE_GC) {
|
||||
// Note: we need to call gc AFTER the actual test so the
|
||||
// gc amount is added to the current test run!
|
||||
commands.gc();
|
||||
}
|
||||
return measureTime(workCallback).then(function(benchmarkData) {
|
||||
var hasGcDuringScript = !!benchmarkData.gcTimeDuringScript;
|
||||
var ignoreBenchmarkRun = false;
|
||||
if (hasGcDuringScript) {
|
||||
gcDuringScriptCount ++;
|
||||
ignoreBenchmarkRun = (mode === MODE_INDETERMINATE || mode === MODE_IGNORE_RUNS_WITH_GC);
|
||||
}).then(function(benchmarkData) {
|
||||
modeState = modeState.nextMode(modeState, benchmarkData, iterationIndex);
|
||||
var action = '';
|
||||
if (modeState.ignoreRun) {
|
||||
action = 'ignore';
|
||||
} else if (modeState.forceGc) {
|
||||
action = 'forceGc';
|
||||
}
|
||||
if (mode === MODE_INDETERMINATE && iterationIndex >= DETERMINE_FORCE_GC_MODE_ITERATIONS) {
|
||||
mode = (gcDuringScriptCount / iterationIndex > 0.5) ? MODE_FORCE_GC : MODE_IGNORE_RUNS_WITH_GC;
|
||||
}
|
||||
|
||||
var rowTitle;
|
||||
if (ignoreBenchmarkRun) {
|
||||
rowTitle = '(ignored: gc in script)';
|
||||
} else {
|
||||
rowTitle = config.logId + '#' + iterationIndex;
|
||||
}
|
||||
reporter.printRow(ROW_FORMAT, [rowTitle]
|
||||
reporter.printRow(ROW_FORMAT, [config.logId + '#' + iterationIndex, action]
|
||||
.concat(formatObjectData(benchmarkData, config.metrics))
|
||||
);
|
||||
|
||||
var benchmarkStats;
|
||||
if (!ignoreBenchmarkRun) {
|
||||
benchmarkStats = benchmarkStatsAggregator(benchmarkData);
|
||||
} else {
|
||||
if (modeState.ignoreRun) {
|
||||
benchmarkStats = benchmarkStatsAggregator.current;
|
||||
} else {
|
||||
benchmarkStats = benchmarkStatsAggregator(benchmarkData);
|
||||
}
|
||||
|
||||
if (Date.now() - startTime > config.timeout) {
|
||||
|
@ -102,22 +143,26 @@ function runBenchmark(config, workCallback) {
|
|||
) {
|
||||
return benchmarkStats
|
||||
}
|
||||
return loop(iterationIndex+1, mode);
|
||||
return loop(iterationIndex+1, modeState);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function formatObjectData(data, props) {
|
||||
return props.map(function(prop) {
|
||||
return data[prop].toFixed(2);
|
||||
var val = data[prop];
|
||||
if (typeof val === 'number') {
|
||||
return val.toFixed(2);
|
||||
} else {
|
||||
return val;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function formatObjectStats(stats, props) {
|
||||
return props.map(function(prop) {
|
||||
var entry = stats[prop];
|
||||
return entry.mean.toFixed(2) + '\u00B1' + entry.coefficientOfVariation.toFixed(2);
|
||||
return entry.mean.toFixed(2) + '\u00B1' + entry.coefficientOfVariation.toFixed(0)+ '%';
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -156,8 +201,8 @@ function sumTimelineRecords(records, startTimeStampId, endTimeStampId) {
|
|||
script: 0,
|
||||
gcTime: 0,
|
||||
gcAmount: 0,
|
||||
gcTimeDuringScript: 0,
|
||||
gcAmountDuringScript: 0,
|
||||
gcTimeInScript: 0,
|
||||
gcAmountInScript: 0,
|
||||
render: 0,
|
||||
timeStamps: []
|
||||
};
|
||||
|
@ -205,8 +250,8 @@ function sumTimelineRecords(records, startTimeStampId, endTimeStampId) {
|
|||
recordStats.gcTime += recordDuration;
|
||||
recordStats.gcAmount += record.data.usedHeapSizeDelta;
|
||||
if (parentIsFunctionCall) {
|
||||
recordStats.gcTimeDuringScript += recordDuration;
|
||||
recordStats.gcAmountDuringScript += record.data.usedHeapSizeDelta;
|
||||
recordStats.gcTimeInScript += recordDuration;
|
||||
recordStats.gcAmountInScript += record.data.usedHeapSizeDelta;
|
||||
}
|
||||
recordUsed = true;
|
||||
} else if (record.type === 'RecalculateStyles' ||
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
var vsprintf = require("sprintf-js").vsprintf;
|
||||
|
||||
var HEADER_SEPARATORS = ['----', '----', '----', '----'];
|
||||
var FOOTER_SEPARATORS = ['====', '====', '====', '===='];
|
||||
var HEADER_SEPARATORS = ['----', '----', '----', '----', '----', '----', '----'];
|
||||
var FOOTER_SEPARATORS = ['====', '====', '====', '====', '====', '====', '===='];
|
||||
|
||||
module.exports = {
|
||||
printHeading: printHeading,
|
||||
|
|
|
@ -5,14 +5,21 @@ module.exports = {
|
|||
calculateStandardDeviation: calculateStandardDeviation
|
||||
};
|
||||
|
||||
function createObjectStatsAggregator(sampleSize) {
|
||||
function createObjectStatsAggregator(properties, sampleSize) {
|
||||
var propSamples = {};
|
||||
var lastResult;
|
||||
addData.current = {};
|
||||
properties.forEach(function(prop) {
|
||||
addData.current[prop] = {
|
||||
mean: 0,
|
||||
coefficientOfVariation: 0,
|
||||
count: 0
|
||||
};
|
||||
});
|
||||
return addData;
|
||||
|
||||
function addData(data) {
|
||||
lastResult = {};
|
||||
for (var prop in data) {
|
||||
var result = {};
|
||||
properties.forEach(function(prop) {
|
||||
var samples = propSamples[prop];
|
||||
if (!samples) {
|
||||
samples = propSamples[prop] = [];
|
||||
|
@ -20,14 +27,14 @@ function createObjectStatsAggregator(sampleSize) {
|
|||
samples.push(data[prop]);
|
||||
samples.splice(0, samples.length - sampleSize);
|
||||
var mean = calculateMean(samples);
|
||||
lastResult[prop] = {
|
||||
result[prop] = {
|
||||
mean: mean,
|
||||
coefficientOfVariation: calculateCoefficientOfVariation(samples, mean),
|
||||
count: samples.length
|
||||
};
|
||||
}
|
||||
addData.current = lastResult;
|
||||
return lastResult;
|
||||
});
|
||||
addData.current = result;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue