refactor(bench press): rename metrics and adapt them to the features of the browser
* Rename metrics, add `Time` suffix to all so that they are more consistent * iOS does not give us `gc` metrics, so they should not be reported * Rename `scriptMicroAvg` into `microScriptTimeAvg` * Rename previous `script` metric into `pureScriptTime` metric, and keep `scriptTime` metric as the overall time, so that we still have a shared metric across devices independent of the supported browser features * `microScriptTimeAvg` is now based on overall `scriptTime`, including gc and render time. * Move more shared DI tokens into `common_options` (previously `sample_options`). Closes #930
This commit is contained in:
Tobias Bosch
parent
75ecaf02b9
commit
21a293b017
|
|
@ -1,14 +1,18 @@
|
|||
import { bind } from 'angular2/di';
|
||||
import { JsonFileReporter } from './common';
|
||||
import { Options } from './common';
|
||||
|
||||
export * from './common';
|
||||
export { SeleniumWebDriverAdapter } from './src/webdriver/selenium_webdriver_adapter';
|
||||
|
||||
var fs = require('fs');
|
||||
|
||||
// Note: Can't do the `require` call in a facade as it can't be loaded into the browser!
|
||||
JsonFileReporter.BINDINGS.push(
|
||||
bind(JsonFileReporter.WRITE_FILE).toValue(writeFile)
|
||||
// TODO(tbosch): right now we bind the `writeFile` method
|
||||
// in benchpres/benchpress.es6. This does not work for Dart,
|
||||
// find another way...
|
||||
// Note: Can't do the `require` call in a facade as it can't be loaded into the browser
|
||||
// for our unit tests via karma.
|
||||
Options.DEFAULT_BINDINGS.push(
|
||||
bind(Options.WRITE_FILE).toValue(writeFile)
|
||||
);
|
||||
|
||||
function writeFile(filename, content) {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ export { Sampler, SampleState } from './src/sampler';
|
|||
export { Metric } from './src/metric';
|
||||
export { Validator } from './src/validator';
|
||||
export { Reporter } from './src/reporter';
|
||||
export { WebDriverExtension } from './src/web_driver_extension';
|
||||
export { WebDriverExtension, PerfLogFeatures } from './src/web_driver_extension';
|
||||
export { WebDriverAdapter } from './src/web_driver_adapter';
|
||||
export { SizeValidator } from './src/validator/size_validator';
|
||||
export { RegressionSlopeValidator } from './src/validator/regression_slope_validator';
|
||||
|
|
@ -13,7 +13,7 @@ export { PerflogMetric } from './src/metric/perflog_metric';
|
|||
export { ChromeDriverExtension } from './src/webdriver/chrome_driver_extension';
|
||||
export { IOsDriverExtension } from './src/webdriver/ios_driver_extension';
|
||||
export { Runner } from './src/runner';
|
||||
export { Options } from './src/sample_options';
|
||||
export { Options } from './src/common_options';
|
||||
export { MeasureValues } from './src/measure_values';
|
||||
export { MultiMetric } from './src/metric/multi_metric';
|
||||
export { MultiReporter } from './src/reporter/multi_reporter';
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import { bind, OpaqueToken } from 'angular2/di';
|
||||
import { DateWrapper } from 'angular2/src/facade/lang';
|
||||
|
||||
export class Options {
|
||||
static get DEFAULT_BINDINGS() { return _DEFAULT_BINDINGS; }
|
||||
// TODO(tbosch): use static initializer when our transpiler supports it
|
||||
static get SAMPLE_ID() { return _SAMPLE_ID; }
|
||||
// TODO(tbosch): use static initializer when our transpiler supports it
|
||||
|
|
@ -23,6 +25,10 @@ export class Options {
|
|||
* Used for micro benchmarks.
|
||||
**/
|
||||
static get MICRO_ITERATIONS() { return _MICRO_ITERATIONS; }
|
||||
// TODO(tbosch): use static initializer when our transpiler supports it
|
||||
static get NOW() { return _NOW; }
|
||||
// TODO(tbosch): use static values when our transpiler supports them
|
||||
static get WRITE_FILE() { return _WRITE_FILE; }
|
||||
}
|
||||
|
||||
var _SAMPLE_ID = new OpaqueToken('Options.sampleId');
|
||||
|
|
@ -34,3 +40,13 @@ var _EXECUTE = new OpaqueToken('Options.execute');
|
|||
var _CAPABILITIES = new OpaqueToken('Options.capabilities');
|
||||
var _USER_AGENT = new OpaqueToken('Options.userAgent');
|
||||
var _MICRO_ITERATIONS = new OpaqueToken('Options.microIterations');
|
||||
var _NOW = new OpaqueToken('Options.now');
|
||||
var _WRITE_FILE = new OpaqueToken('Options.writeFile');
|
||||
|
||||
var _DEFAULT_BINDINGS = [
|
||||
bind(_DEFAULT_DESCRIPTION).toValue({}),
|
||||
bind(_SAMPLE_DESCRIPTION).toValue({}),
|
||||
bind(_FORCE_GC).toValue(false),
|
||||
bind(_PREPARE).toValue(false),
|
||||
bind(_NOW).toValue( () => DateWrapper.now() )
|
||||
];
|
||||
|
|
@ -3,9 +3,9 @@ import { isPresent, isBlank, int, BaseException, StringWrapper, Math } from 'ang
|
|||
import { ListWrapper, StringMap, StringMapWrapper } from 'angular2/src/facade/collection';
|
||||
import { bind, OpaqueToken } from 'angular2/di';
|
||||
|
||||
import { WebDriverExtension } from '../web_driver_extension';
|
||||
import { WebDriverExtension, PerfLogFeatures } from '../web_driver_extension';
|
||||
import { Metric } from '../metric';
|
||||
import { Options } from '../sample_options';
|
||||
import { Options } from '../common_options';
|
||||
|
||||
/**
|
||||
* A metric that reads out the performance log
|
||||
|
|
@ -21,6 +21,7 @@ export class PerflogMetric extends Metric {
|
|||
_measureCount:int;
|
||||
_setTimeout:Function;
|
||||
_microIterations:int;
|
||||
_perfLogFeatures:PerfLogFeatures;
|
||||
|
||||
/**
|
||||
* @param driverExtension
|
||||
|
|
@ -35,19 +36,24 @@ export class PerflogMetric extends Metric {
|
|||
this._measureCount = 0;
|
||||
this._setTimeout = setTimeout;
|
||||
this._microIterations = microIterations;
|
||||
this._perfLogFeatures = driverExtension.perfLogFeatures();
|
||||
}
|
||||
|
||||
describe():StringMap {
|
||||
var res = {
|
||||
'script': 'script execution time in ms',
|
||||
'render': 'render time in ms',
|
||||
'gcTime': 'gc time in ms',
|
||||
'gcAmount': 'gc amount in kbytes',
|
||||
'majorGcTime': 'time of major gcs in ms',
|
||||
'majorGcAmount': 'amount of major gcs in kbytes'
|
||||
'scriptTime': 'script execution time in ms, including gc and render',
|
||||
'pureScriptTime': 'script execution time in ms, without gc nor render'
|
||||
};
|
||||
if (this._perfLogFeatures.render) {
|
||||
res['renderTime'] = 'render time in and ouside of script in ms';
|
||||
}
|
||||
if (this._perfLogFeatures.gc) {
|
||||
res['gcTime'] = 'gc time in and ouside of script in ms';
|
||||
res['gcAmount'] = 'gc amount in kbytes';
|
||||
res['majorGcTime'] = 'time of major gcs in ms';
|
||||
}
|
||||
if (this._microIterations > 0) {
|
||||
res['scriptMicroAvg'] = 'average script time for a micro iteration';
|
||||
res['microScriptTimeAvg'] = 'average script time for a micro iteration';
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
|
@ -120,17 +126,22 @@ export class PerflogMetric extends Metric {
|
|||
|
||||
_aggregateEvents(events, markName) {
|
||||
var result = {
|
||||
'script': 0,
|
||||
'render': 0,
|
||||
'gcTime': 0,
|
||||
'gcAmount': 0,
|
||||
'majorGcTime': 0,
|
||||
'majorGcAmount': 0
|
||||
'scriptTime': 0,
|
||||
'pureScriptTime': 0
|
||||
};
|
||||
if (this._perfLogFeatures.gc) {
|
||||
result['gcTime'] = 0;
|
||||
result['majorGcTime'] = 0;
|
||||
result['gcAmount'] = 0;
|
||||
}
|
||||
if (this._perfLogFeatures.render) {
|
||||
result['renderTime'] = 0;
|
||||
}
|
||||
|
||||
var markStartEvent = null;
|
||||
var markEndEvent = null;
|
||||
var gcTimeInScript = 0;
|
||||
var renderTimeInScript = 0;
|
||||
|
||||
var intervalStarts = {};
|
||||
events.forEach( (event) => {
|
||||
|
|
@ -149,26 +160,30 @@ export class PerflogMetric extends Metric {
|
|||
var duration = event['ts'] - startEvent['ts'];
|
||||
intervalStarts[name] = null;
|
||||
if (StringWrapper.equals(name, 'gc')) {
|
||||
var amount = (startEvent['args']['usedHeapSize'] - event['args']['usedHeapSize']) / 1000;
|
||||
result['gcTime'] += duration;
|
||||
var amount = (startEvent['args']['usedHeapSize'] - event['args']['usedHeapSize']) / 1000;
|
||||
result['gcAmount'] += amount;
|
||||
var majorGc = event['args']['majorGc'];
|
||||
if (isPresent(majorGc) && majorGc) {
|
||||
result['majorGcTime'] += duration;
|
||||
result['majorGcAmount'] += amount;
|
||||
}
|
||||
if (isPresent(intervalStarts['script'])) {
|
||||
gcTimeInScript += duration;
|
||||
}
|
||||
} else if (StringWrapper.equals(name, 'script') || StringWrapper.equals(name, 'render')) {
|
||||
result[name] += duration;
|
||||
} else if (StringWrapper.equals(name, 'render')) {
|
||||
result['renderTime'] += duration;
|
||||
if (isPresent(intervalStarts['script'])) {
|
||||
renderTimeInScript += duration;
|
||||
}
|
||||
} else if (StringWrapper.equals(name, 'script')) {
|
||||
result['scriptTime'] += duration;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
result['script'] -= gcTimeInScript;
|
||||
result['pureScriptTime'] = result['scriptTime'] - gcTimeInScript - renderTimeInScript;
|
||||
if (this._microIterations > 0) {
|
||||
result['scriptMicroAvg'] = result['script'] / this._microIterations;
|
||||
result['microScriptTimeAvg'] = result['scriptTime'] / this._microIterations;
|
||||
}
|
||||
return isPresent(markStartEvent) && isPresent(markEndEvent) ? result : null;
|
||||
}
|
||||
|
|
@ -183,7 +198,8 @@ var _MARK_NAME_PREFIX = 'benchpress';
|
|||
var _SET_TIMEOUT = new OpaqueToken('PerflogMetric.setTimeout');
|
||||
var _BINDINGS = [
|
||||
bind(PerflogMetric).toFactory(
|
||||
(driverExtension, setTimeout, microIterations) => new PerflogMetric(driverExtension, setTimeout, microIterations),
|
||||
(driverExtension, setTimeout, microIterations) =>
|
||||
new PerflogMetric(driverExtension, setTimeout, microIterations),
|
||||
[WebDriverExtension, _SET_TIMEOUT, Options.MICRO_ITERATIONS]
|
||||
),
|
||||
bind(_SET_TIMEOUT).toValue( (fn, millis) => PromiseWrapper.setTimeout(fn, millis) ),
|
||||
|
|
|
|||
|
|
@ -7,16 +7,12 @@ import { bind, OpaqueToken } from 'angular2/di';
|
|||
import { Reporter } from '../reporter';
|
||||
import { SampleDescription } from '../sample_description';
|
||||
import { MeasureValues } from '../measure_values';
|
||||
import { Options } from '../common_options';
|
||||
|
||||
/**
|
||||
* A reporter that writes results into a json file.
|
||||
* TODO(tbosch): right now we bind the `writeFile` method
|
||||
* in benchpres/benchpress.es6. This does not work for Dart,
|
||||
* find another way...
|
||||
*/
|
||||
export class JsonFileReporter extends Reporter {
|
||||
// TODO(tbosch): use static values when our transpiler supports them
|
||||
static get WRITE_FILE() { return _WRITE_FILE; }
|
||||
// TODO(tbosch): use static values when our transpiler supports them
|
||||
static get PATH() { return _PATH; }
|
||||
static get BINDINGS() { return _BINDINGS; }
|
||||
|
|
@ -24,12 +20,14 @@ export class JsonFileReporter extends Reporter {
|
|||
_writeFile:Function;
|
||||
_path:string;
|
||||
_description:SampleDescription;
|
||||
_now:Function;
|
||||
|
||||
constructor(sampleDescription, path, writeFile) {
|
||||
constructor(sampleDescription, path, writeFile, now) {
|
||||
super();
|
||||
this._description = sampleDescription;
|
||||
this._path = path;
|
||||
this._writeFile = writeFile;
|
||||
this._now = now;
|
||||
}
|
||||
|
||||
reportMeasureValues(measureValues:MeasureValues):Promise {
|
||||
|
|
@ -42,17 +40,16 @@ export class JsonFileReporter extends Reporter {
|
|||
'completeSample': completeSample,
|
||||
'validSample': validSample
|
||||
});
|
||||
var filePath = `${this._path}/${this._description.id}_${DateWrapper.toMillis(DateWrapper.now())}.json`;
|
||||
var filePath = `${this._path}/${this._description.id}_${DateWrapper.toMillis(this._now())}.json`;
|
||||
return this._writeFile(filePath, content);
|
||||
}
|
||||
}
|
||||
|
||||
var _WRITE_FILE = new OpaqueToken('JsonFileReporter.writeFile');
|
||||
var _PATH = new OpaqueToken('JsonFileReporter.path');
|
||||
var _BINDINGS = [
|
||||
bind(JsonFileReporter).toFactory(
|
||||
(sampleDescription, path, writeFile) => new JsonFileReporter(sampleDescription, path, writeFile),
|
||||
[SampleDescription, _PATH, _WRITE_FILE]
|
||||
(sampleDescription, path, writeFile, now) => new JsonFileReporter(sampleDescription, path, writeFile, now),
|
||||
[SampleDescription, _PATH, Options.WRITE_FILE, Options.NOW]
|
||||
),
|
||||
bind(_PATH).toValue('.')
|
||||
];
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import { SampleDescription } from './sample_description';
|
|||
import { WebDriverAdapter } from './web_driver_adapter';
|
||||
import { Reporter } from './reporter';
|
||||
import { Metric } from './metric';
|
||||
import { Options } from './sample_options';
|
||||
import { Options } from './common_options';
|
||||
|
||||
/**
|
||||
* The Runner is the main entry point for executing a sample run.
|
||||
|
|
@ -56,6 +56,7 @@ export class Runner {
|
|||
}
|
||||
|
||||
var _DEFAULT_BINDINGS = [
|
||||
Options.DEFAULT_BINDINGS,
|
||||
Sampler.BINDINGS,
|
||||
ConsoleReporter.BINDINGS,
|
||||
RegressionSlopeValidator.BINDINGS,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { StringMapWrapper, ListWrapper, StringMap } from 'angular2/src/facade/co
|
|||
import { bind, OpaqueToken } from 'angular2/di';
|
||||
import { Validator } from './validator';
|
||||
import { Metric } from './metric';
|
||||
import { Options } from './sample_options';
|
||||
import { Options } from './common_options';
|
||||
|
||||
/**
|
||||
* SampleDescription merges all available descriptions about a sample
|
||||
|
|
@ -47,7 +47,5 @@ var _BINDINGS = [
|
|||
Metric, Options.SAMPLE_ID, Options.FORCE_GC, Options.USER_AGENT,
|
||||
Validator, Options.DEFAULT_DESCRIPTION, Options.SAMPLE_DESCRIPTION
|
||||
]
|
||||
),
|
||||
bind(Options.DEFAULT_DESCRIPTION).toValue({}),
|
||||
bind(Options.SAMPLE_DESCRIPTION).toValue({})
|
||||
)
|
||||
];
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { Reporter } from './reporter';
|
|||
import { WebDriverExtension } from './web_driver_extension';
|
||||
import { WebDriverAdapter } from './web_driver_adapter';
|
||||
|
||||
import { Options } from './sample_options';
|
||||
import { Options } from './common_options';
|
||||
import { MeasureValues} from './measure_values';
|
||||
|
||||
/**
|
||||
|
|
@ -23,8 +23,6 @@ import { MeasureValues} from './measure_values';
|
|||
export class Sampler {
|
||||
// TODO(tbosch): use static values when our transpiler supports them
|
||||
static get BINDINGS() { return _BINDINGS; }
|
||||
// TODO(tbosch): use static values when our transpiler supports them
|
||||
static get TIME() { return _TIME; }
|
||||
|
||||
_driver:WebDriverAdapter;
|
||||
_driverExtension:WebDriverExtension;
|
||||
|
|
@ -34,14 +32,14 @@ export class Sampler {
|
|||
_forceGc:boolean;
|
||||
_prepare:Function;
|
||||
_execute:Function;
|
||||
_time:Function;
|
||||
_now:Function;
|
||||
|
||||
constructor({
|
||||
driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, time
|
||||
driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, now
|
||||
}:{
|
||||
driver: WebDriverAdapter,
|
||||
driverExtension: WebDriverExtension, metric: Metric, reporter: Reporter,
|
||||
validator: Validator, prepare: Function, execute: Function, time: Function
|
||||
validator: Validator, prepare: Function, execute: Function, now: Function
|
||||
}={}) {
|
||||
this._driver = driver;
|
||||
this._driverExtension = driverExtension;
|
||||
|
|
@ -51,7 +49,7 @@ export class Sampler {
|
|||
this._forceGc = forceGc;
|
||||
this._prepare = prepare;
|
||||
this._execute = execute;
|
||||
this._time = time;
|
||||
this._now = now;
|
||||
}
|
||||
|
||||
sample():Promise<SampleState> {
|
||||
|
|
@ -96,7 +94,7 @@ export class Sampler {
|
|||
}
|
||||
|
||||
_report(state:SampleState, metricValues:StringMap):Promise<SampleState> {
|
||||
var measureValues = new MeasureValues(state.completeSample.length, this._time(), metricValues);
|
||||
var measureValues = new MeasureValues(state.completeSample.length, this._now(), metricValues);
|
||||
var completeSample = ListWrapper.concat(state.completeSample, [measureValues]);
|
||||
var validSample = this._validator.validate(completeSample);
|
||||
var resultPromise = this._reporter.reportMeasureValues(measureValues);
|
||||
|
|
@ -118,11 +116,9 @@ export class SampleState {
|
|||
}
|
||||
}
|
||||
|
||||
var _TIME = new OpaqueToken('Sampler.time');
|
||||
|
||||
var _BINDINGS = [
|
||||
bind(Sampler).toFactory(
|
||||
(driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, time) => new Sampler({
|
||||
(driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, now) => new Sampler({
|
||||
driver: driver,
|
||||
driverExtension: driverExtension,
|
||||
reporter: reporter,
|
||||
|
|
@ -134,14 +130,11 @@ var _BINDINGS = [
|
|||
// special null object, which is expensive.
|
||||
prepare: prepare !== false ? prepare : null,
|
||||
execute: execute,
|
||||
time: time
|
||||
now: now
|
||||
}),
|
||||
[
|
||||
WebDriverAdapter, WebDriverExtension, Metric, Reporter, Validator,
|
||||
Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, _TIME
|
||||
Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, Options.NOW
|
||||
]
|
||||
),
|
||||
bind(Options.FORCE_GC).toValue(false),
|
||||
bind(Options.PREPARE).toValue(false),
|
||||
bind(_TIME).toValue( () => DateWrapper.now() )
|
||||
)
|
||||
];
|
||||
|
|
|
|||
|
|
@ -65,5 +65,5 @@ var _BINDINGS = [
|
|||
[_SAMPLE_SIZE, _METRIC]
|
||||
),
|
||||
bind(_SAMPLE_SIZE).toValue(10),
|
||||
bind(_METRIC).toValue('script')
|
||||
bind(_METRIC).toValue('scriptTime')
|
||||
];
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import { bind, Injector, OpaqueToken } from 'angular2/di';
|
||||
|
||||
import { BaseException, ABSTRACT, isBlank } from 'angular2/src/facade/lang';
|
||||
import { BaseException, ABSTRACT, isBlank, isPresent } from 'angular2/src/facade/lang';
|
||||
import { Promise, PromiseWrapper } from 'angular2/src/facade/async';
|
||||
import { List, ListWrapper, StringMap } from 'angular2/src/facade/collection';
|
||||
|
||||
import { Options } from './sample_options';
|
||||
import { Options } from './common_options';
|
||||
|
||||
/**
|
||||
* A WebDriverExtension implements extended commands of the webdriver protocol
|
||||
|
|
@ -64,9 +64,23 @@ export class WebDriverExtension {
|
|||
throw new BaseException('NYI');
|
||||
}
|
||||
|
||||
perfLogFeatures():PerfLogFeatures {
|
||||
throw new BaseException('NYI');
|
||||
}
|
||||
|
||||
supports(capabilities:StringMap):boolean {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export class PerfLogFeatures {
|
||||
render:boolean;
|
||||
gc:boolean;
|
||||
|
||||
constructor({render, gc} = {}) {
|
||||
this.render = isPresent(render) && render;
|
||||
this.gc = isPresent(gc) && gc;
|
||||
}
|
||||
}
|
||||
|
||||
var _CHILDREN = new OpaqueToken('WebDriverExtension.children');
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import {
|
|||
Json, isPresent, isBlank, RegExpWrapper, StringWrapper, BaseException, NumberWrapper
|
||||
} from 'angular2/src/facade/lang';
|
||||
|
||||
import { WebDriverExtension } from '../web_driver_extension';
|
||||
import { WebDriverExtension, PerfLogFeatures } from '../web_driver_extension';
|
||||
import { WebDriverAdapter } from '../web_driver_adapter';
|
||||
import { Promise } from 'angular2/src/facade/async';
|
||||
|
||||
|
|
@ -111,6 +111,13 @@ export class ChromeDriverExtension extends WebDriverExtension {
|
|||
return normalizedEvents;
|
||||
}
|
||||
|
||||
perfLogFeatures():PerfLogFeatures {
|
||||
return new PerfLogFeatures({
|
||||
render: true,
|
||||
gc: true
|
||||
});
|
||||
}
|
||||
|
||||
supports(capabilities:StringMap):boolean {
|
||||
return StringWrapper.equals(capabilities['browserName'].toLowerCase(), 'chrome');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import { bind } from 'angular2/di';
|
||||
import { ListWrapper, StringMap } from 'angular2/src/facade/collection';
|
||||
import {
|
||||
Json, isPresent, isBlank, RegExpWrapper, StringWrapper
|
||||
Json, isPresent, isBlank, RegExpWrapper, StringWrapper, BaseException
|
||||
} from 'angular2/src/facade/lang';
|
||||
|
||||
import { WebDriverExtension } from '../web_driver_extension';
|
||||
import { WebDriverExtension, PerfLogFeatures } from '../web_driver_extension';
|
||||
import { WebDriverAdapter } from '../web_driver_adapter';
|
||||
import { Promise } from 'angular2/src/facade/async';
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ export class IOsDriverExtension extends WebDriverExtension {
|
|||
}
|
||||
|
||||
gc() {
|
||||
return this._driver.executeScript('window.gc()');
|
||||
throw new BaseException('Force GC is not supported on iOS');
|
||||
}
|
||||
|
||||
timeBegin(name:string):Promise {
|
||||
|
|
@ -81,14 +81,8 @@ export class IOsDriverExtension extends WebDriverExtension {
|
|||
StringWrapper.equals(type, 'CompositeLayers')) {
|
||||
ListWrapper.push(events, createStartEvent('render', startTime));
|
||||
endEvent = createEndEvent('render', endTime);
|
||||
} else if (StringWrapper.equals(type, 'GCEvent')) {
|
||||
ListWrapper.push(events, createStartEvent('gc', startTime, {
|
||||
'usedHeapSize': 0
|
||||
}));
|
||||
endEvent = createEndEvent('gc', endTime, {
|
||||
'usedHeapSize': -data['usedHeapSizeDelta']
|
||||
});
|
||||
}
|
||||
// Note: ios used to support GCEvent up until iOS 6 :-(
|
||||
if (isPresent(record['children'])) {
|
||||
this._convertPerfRecordsToEvents(record['children'], events);
|
||||
}
|
||||
|
|
@ -99,6 +93,12 @@ export class IOsDriverExtension extends WebDriverExtension {
|
|||
return events;
|
||||
}
|
||||
|
||||
perfLogFeatures():PerfLogFeatures {
|
||||
return new PerfLogFeatures({
|
||||
render: true
|
||||
});
|
||||
}
|
||||
|
||||
supports(capabilities:StringMap):boolean {
|
||||
return StringWrapper.equals(capabilities['browserName'].toLowerCase(), 'safari');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,9 +13,13 @@ import {
|
|||
|
||||
import { List, ListWrapper } from 'angular2/src/facade/collection';
|
||||
import { PromiseWrapper, Promise } from 'angular2/src/facade/async';
|
||||
import { isPresent } from 'angular2/src/facade/lang';
|
||||
import { isPresent, isBlank } from 'angular2/src/facade/lang';
|
||||
|
||||
import { Metric, PerflogMetric, WebDriverExtension, bind, Injector, Options } from 'benchpress/common';
|
||||
import {
|
||||
Metric, PerflogMetric, WebDriverExtension,
|
||||
PerfLogFeatures,
|
||||
bind, Injector, Options
|
||||
} from 'benchpress/common';
|
||||
|
||||
import { TraceEventFactory } from '../trace_event_factory';
|
||||
|
||||
|
|
@ -23,15 +27,19 @@ export function main() {
|
|||
var commandLog;
|
||||
var eventFactory = new TraceEventFactory('timeline', 'pid0');
|
||||
|
||||
function createMetric(perfLogs, microIterations = 0) {
|
||||
function createMetric(perfLogs, microIterations = 0, perfLogFeatures = null) {
|
||||
commandLog = [];
|
||||
if (isBlank(perfLogFeatures)) {
|
||||
perfLogFeatures = new PerfLogFeatures({render: true, gc: true});
|
||||
}
|
||||
var bindings = [
|
||||
Options.DEFAULT_BINDINGS,
|
||||
PerflogMetric.BINDINGS,
|
||||
bind(PerflogMetric.SET_TIMEOUT).toValue( (fn, millis) => {
|
||||
ListWrapper.push(commandLog, ['setTimeout', millis]);
|
||||
fn();
|
||||
}),
|
||||
bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog)),
|
||||
bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog, perfLogFeatures)),
|
||||
bind(Options.MICRO_ITERATIONS).toValue(microIterations)
|
||||
];
|
||||
return new Injector(bindings).get(PerflogMetric);
|
||||
|
|
@ -39,8 +47,29 @@ export function main() {
|
|||
|
||||
describe('perflog metric', () => {
|
||||
|
||||
it('should describe itself', () => {
|
||||
expect(createMetric([[]]).describe()['script']).toBe('script execution time in ms');
|
||||
it('should describe itself based on the perfLogFeatrues', () => {
|
||||
expect(createMetric([[]], 0, new PerfLogFeatures()).describe()).toEqual({
|
||||
'scriptTime': 'script execution time in ms, including gc and render',
|
||||
'pureScriptTime': 'script execution time in ms, without gc nor render'
|
||||
});
|
||||
|
||||
expect(createMetric([[]], 0, new PerfLogFeatures({
|
||||
render: true,
|
||||
gc: false
|
||||
})).describe()).toEqual({
|
||||
'scriptTime': 'script execution time in ms, including gc and render',
|
||||
'pureScriptTime': 'script execution time in ms, without gc nor render',
|
||||
'renderTime': 'render time in and ouside of script in ms',
|
||||
});
|
||||
|
||||
expect(createMetric([[]]).describe()).toEqual({
|
||||
'scriptTime': 'script execution time in ms, including gc and render',
|
||||
'pureScriptTime': 'script execution time in ms, without gc nor render',
|
||||
'renderTime': 'render time in and ouside of script in ms',
|
||||
'gcTime': 'gc time in and ouside of script in ms',
|
||||
'gcAmount': 'gc amount in kbytes',
|
||||
'majorGcTime': 'time of major gcs in ms'
|
||||
});
|
||||
});
|
||||
|
||||
describe('beginMeasure', () => {
|
||||
|
|
@ -76,7 +105,7 @@ export function main() {
|
|||
['timeEnd', 'benchpress0', null],
|
||||
'readPerfLog'
|
||||
]);
|
||||
expect(data['script']).toBe(2);
|
||||
expect(data['scriptTime']).toBe(2);
|
||||
|
||||
async.done();
|
||||
});
|
||||
|
|
@ -128,7 +157,7 @@ export function main() {
|
|||
[ 'setTimeout', 100 ],
|
||||
'readPerfLog'
|
||||
]);
|
||||
expect(data['script']).toBe(3);
|
||||
expect(data['scriptTime']).toBe(3);
|
||||
|
||||
async.done();
|
||||
});
|
||||
|
|
@ -144,7 +173,7 @@ export function main() {
|
|||
metric.beginMeasure()
|
||||
.then( (_) => metric.endMeasure(true) )
|
||||
.then( (data) => {
|
||||
expect(data['script']).toBe(0);
|
||||
expect(data['scriptTime']).toBe(0);
|
||||
return metric.endMeasure(true)
|
||||
})
|
||||
.then( (data) => {
|
||||
|
|
@ -155,7 +184,7 @@ export function main() {
|
|||
['timeEnd', 'benchpress1', 'benchpress2'],
|
||||
'readPerfLog'
|
||||
]);
|
||||
expect(data['script']).toBe(3);
|
||||
expect(data['scriptTime']).toBe(3);
|
||||
|
||||
async.done();
|
||||
});
|
||||
|
|
@ -179,7 +208,7 @@ export function main() {
|
|||
eventFactory.start('script', 0),
|
||||
eventFactory.end('script', 5)
|
||||
]).then((data) => {
|
||||
expect(data['script']).toBe(5);
|
||||
expect(data['scriptTime']).toBe(5);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
|
@ -191,7 +220,7 @@ export function main() {
|
|||
eventFactory.start('script', 10),
|
||||
eventFactory.end('script', 17)
|
||||
]).then((data) => {
|
||||
expect(data['script']).toBe(12);
|
||||
expect(data['scriptTime']).toBe(12);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
|
@ -200,7 +229,7 @@ export function main() {
|
|||
aggregate([
|
||||
eventFactory.end('script', 10)
|
||||
]).then((data) => {
|
||||
expect(data['script']).toBe(0);
|
||||
expect(data['scriptTime']).toBe(0);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
|
@ -209,7 +238,7 @@ export function main() {
|
|||
aggregate([
|
||||
eventFactory.start('script', 10)
|
||||
]).then((data) => {
|
||||
expect(data['script']).toBe(0);
|
||||
expect(data['scriptTime']).toBe(0);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
|
@ -227,22 +256,30 @@ export function main() {
|
|||
metric.beginMeasure()
|
||||
.then( (_) => metric.endMeasure(false) )
|
||||
.then((data) => {
|
||||
expect(data['script']).toBe(5);
|
||||
expect(data['scriptTime']).toBe(5);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
||||
['script', 'render'].forEach( (metricName) => {
|
||||
it(`should support ${metricName} metric`, inject([AsyncTestCompleter], (async) => {
|
||||
it('should support scriptTime metric', inject([AsyncTestCompleter], (async) => {
|
||||
aggregate([
|
||||
eventFactory.start(metricName, 0),
|
||||
eventFactory.end(metricName, 5)
|
||||
eventFactory.start('script', 0),
|
||||
eventFactory.end('script', 5)
|
||||
]).then((data) => {
|
||||
expect(data[metricName]).toBe(5);
|
||||
expect(data['scriptTime']).toBe(5);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
||||
it('should support renderTime metric', inject([AsyncTestCompleter], (async) => {
|
||||
aggregate([
|
||||
eventFactory.start('render', 0),
|
||||
eventFactory.end('render', 5)
|
||||
]).then((data) => {
|
||||
expect(data['renderTime']).toBe(5);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
||||
it('should support gcTime/gcAmount metric', inject([AsyncTestCompleter], (async) => {
|
||||
aggregate([
|
||||
|
|
@ -252,55 +289,55 @@ export function main() {
|
|||
expect(data['gcTime']).toBe(5);
|
||||
expect(data['gcAmount']).toBe(1.5);
|
||||
expect(data['majorGcTime']).toBe(0);
|
||||
expect(data['majorGcAmount']).toBe(0);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
||||
it('should support majorGcTime/majorGcAmount metric', inject([AsyncTestCompleter], (async) => {
|
||||
it('should support majorGcTime metric', inject([AsyncTestCompleter], (async) => {
|
||||
aggregate([
|
||||
eventFactory.start('gc', 0, {'usedHeapSize': 2500}),
|
||||
eventFactory.end('gc', 5, {'usedHeapSize': 1000, 'majorGc': true})
|
||||
]).then((data) => {
|
||||
expect(data['gcTime']).toBe(5);
|
||||
expect(data['gcAmount']).toBe(1.5);
|
||||
expect(data['majorGcTime']).toBe(5);
|
||||
expect(data['majorGcAmount']).toBe(1.5);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
||||
it('should subtract gcTime in script from script time', inject([AsyncTestCompleter], (async) => {
|
||||
it('should support pureScriptTime = scriptTime-gcTime-renderTime', inject([AsyncTestCompleter], (async) => {
|
||||
aggregate([
|
||||
eventFactory.start('script', 0),
|
||||
eventFactory.start('gc', 1, {'usedHeapSize': 1000}),
|
||||
eventFactory.end('gc', 4, {'usedHeapSize': 0}),
|
||||
eventFactory.end('script', 5)
|
||||
eventFactory.start('render', 4),
|
||||
eventFactory.end('render', 5),
|
||||
eventFactory.end('script', 6)
|
||||
]).then((data) => {
|
||||
expect(data['script']).toBe(2);
|
||||
expect(data['scriptTime']).toBe(6);
|
||||
expect(data['pureScriptTime']).toBe(2);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
||||
describe('microIterations', () => {
|
||||
|
||||
it('should not report scriptMicroAvg if microIterations = 0', inject([AsyncTestCompleter], (async) => {
|
||||
it('should not report microScriptTimeAvg if microIterations = 0', inject([AsyncTestCompleter], (async) => {
|
||||
aggregate([
|
||||
eventFactory.start('script', 0),
|
||||
eventFactory.end('script', 5)
|
||||
], 0).then((data) => {
|
||||
expect(isPresent(data['scriptMicroAvg'])).toBe(false);
|
||||
expect(isPresent(data['microScriptTimeAvg'])).toBe(false);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
||||
it('should report scriptMicroAvg', inject([AsyncTestCompleter], (async) => {
|
||||
it('should report microScriptTimeAvg', inject([AsyncTestCompleter], (async) => {
|
||||
aggregate([
|
||||
eventFactory.start('script', 0),
|
||||
eventFactory.end('script', 5)
|
||||
], 4).then((data) => {
|
||||
expect(data['script']).toBe(5);
|
||||
expect(data['scriptMicroAvg']).toBe(5/4);
|
||||
expect(data['scriptTime']).toBe(5);
|
||||
expect(data['microScriptTimeAvg']).toBe(5/4);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
|
@ -315,10 +352,12 @@ export function main() {
|
|||
class MockDriverExtension extends WebDriverExtension {
|
||||
_perfLogs:List;
|
||||
_commandLog:List;
|
||||
constructor(perfLogs, commandLog) {
|
||||
_perfLogFeatures:PerfLogFeatures;
|
||||
constructor(perfLogs, commandLog, perfLogFeatures) {
|
||||
super();
|
||||
this._perfLogs = perfLogs;
|
||||
this._commandLog = commandLog;
|
||||
this._perfLogFeatures = perfLogFeatures;
|
||||
}
|
||||
|
||||
timeBegin(name):Promise {
|
||||
|
|
@ -331,6 +370,10 @@ class MockDriverExtension extends WebDriverExtension {
|
|||
return PromiseWrapper.resolve(null);
|
||||
}
|
||||
|
||||
perfLogFeatures():PerfLogFeatures {
|
||||
return this._perfLogFeatures;
|
||||
}
|
||||
|
||||
readPerfLog():Promise {
|
||||
ListWrapper.push(this._commandLog, 'readPerfLog');
|
||||
if (this._perfLogs.length > 0) {
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ import { PromiseWrapper } from 'angular2/src/facade/async';
|
|||
import {
|
||||
bind, Injector,
|
||||
SampleDescription,
|
||||
MeasureValues
|
||||
MeasureValues,
|
||||
Options
|
||||
} from 'benchpress/common';
|
||||
|
||||
|
||||
|
|
@ -32,7 +33,8 @@ export function main() {
|
|||
JsonFileReporter.BINDINGS,
|
||||
bind(SampleDescription).toValue(new SampleDescription(sampleId, descriptions, metrics)),
|
||||
bind(JsonFileReporter.PATH).toValue(path),
|
||||
bind(JsonFileReporter.WRITE_FILE).toValue((filename, content) => {
|
||||
bind(Options.NOW).toValue( () => DateWrapper.fromMillis(1234) ),
|
||||
bind(Options.WRITE_FILE).toValue((filename, content) => {
|
||||
loggedFile = {
|
||||
'filename': filename,
|
||||
'content': content
|
||||
|
|
|
|||
|
|
@ -50,15 +50,17 @@ export function main() {
|
|||
if (isBlank(driverExtension)) {
|
||||
driverExtension = new MockDriverExtension([]);
|
||||
}
|
||||
var bindings = ListWrapper.concat(Sampler.BINDINGS, [
|
||||
var bindings = [
|
||||
Options.DEFAULT_BINDINGS,
|
||||
Sampler.BINDINGS,
|
||||
bind(Metric).toValue(metric),
|
||||
bind(Reporter).toValue(reporter),
|
||||
bind(WebDriverAdapter).toValue(driver),
|
||||
bind(WebDriverExtension).toValue(driverExtension),
|
||||
bind(Options.EXECUTE).toValue(execute),
|
||||
bind(Validator).toValue(validator),
|
||||
bind(Sampler.TIME).toValue( () => DateWrapper.fromMillis(time++) )
|
||||
]);
|
||||
bind(Options.NOW).toValue( () => DateWrapper.fromMillis(time++) )
|
||||
];
|
||||
if (isPresent(prepare)) {
|
||||
ListWrapper.push(bindings, bind(Options.PREPARE).toValue(prepare));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,12 +41,9 @@ export function main() {
|
|||
return extension;
|
||||
}
|
||||
|
||||
it('should force gc via window.gc()', inject([AsyncTestCompleter], (async) => {
|
||||
createExtension().gc().then( (_) => {
|
||||
expect(log).toEqual([['executeScript', 'window.gc()']]);
|
||||
async.done();
|
||||
it('should throw on forcing gc', () => {
|
||||
expect( () => createExtension().gc() ).toThrowError('Force GC is not supported on iOS');
|
||||
});
|
||||
}));
|
||||
|
||||
it('should mark the timeline via console.time()', inject([AsyncTestCompleter], (async) => {
|
||||
createExtension().timeBegin('someName').then( (_) => {
|
||||
|
|
@ -124,18 +121,6 @@ export function main() {
|
|||
});
|
||||
}));
|
||||
|
||||
it('should report gc', inject([AsyncTestCompleter], (async) => {
|
||||
createExtension([
|
||||
gcRecord(1, 3, 21)
|
||||
]).readPerfLog().then( (events) => {
|
||||
expect(events).toEqual([
|
||||
normEvents.start('gc', 1, {'usedHeapSize': 0}),
|
||||
normEvents.end('gc', 3, {'usedHeapSize': -21}),
|
||||
]);
|
||||
async.done();
|
||||
});
|
||||
}));
|
||||
|
||||
['RecalculateStyles', 'Layout', 'UpdateLayerTree', 'Paint', 'Rasterize', 'CompositeLayers'].forEach( (recordType) => {
|
||||
it(`should report ${recordType}`, inject([AsyncTestCompleter], (async) => {
|
||||
createExtension([
|
||||
|
|
@ -224,17 +209,6 @@ function internalScriptRecord(startTime, endTime) {
|
|||
};
|
||||
}
|
||||
|
||||
function gcRecord(startTime, endTime, gcAmount) {
|
||||
return {
|
||||
'type': 'GCEvent',
|
||||
'startTime': startTime,
|
||||
'endTime': endTime,
|
||||
'data': {
|
||||
'usedHeapSizeDelta': gcAmount
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
class MockDriverAdapter extends WebDriverAdapter {
|
||||
_log:List;
|
||||
_perfRecords:List;
|
||||
|
|
|
|||
Loading…
Reference in New Issue