From 21a293b01771e25bf38a5372b1e74dc2fa767b8a Mon Sep 17 00:00:00 2001 From: Tobias Bosch Date: Fri, 6 Mar 2015 17:34:27 -0800 Subject: [PATCH] refactor(bench press): rename metrics and adapt them to the features of the browser * Rename metrics, add `Time` suffix to all so that they are more consistent * iOS does not give us `gc` metrics, so they should not be reported * Rename `scriptMicroAvg` into `microScriptTimeAvg` * Rename previous `script` metric into `pureScriptTime` metric, and keep `scriptTime` metric as the overall time, so that we still have a shared metric across devices independent of the supported browser features * `microScriptTimeAvg` is now based on overall `scriptTime`, including gc and render time. * Move more shared DI tokens into `common_options` (previously `sample_options`). Closes #930 --- modules/benchpress/benchpress.es6 | 12 +- modules/benchpress/common.js | 4 +- .../{sample_options.js => common_options.js} | 16 +++ .../benchpress/src/metric/perflog_metric.js | 60 +++++---- .../src/reporter/json_file_reporter.js | 17 +-- modules/benchpress/src/runner.js | 3 +- modules/benchpress/src/sample_description.js | 6 +- modules/benchpress/src/sampler.js | 27 ++-- .../validator/regression_slope_validator.js | 2 +- .../benchpress/src/web_driver_extension.js | 18 ++- .../src/webdriver/chrome_driver_extension.js | 9 +- .../src/webdriver/ios_driver_extension.js | 20 +-- .../test/metric/perflog_metric_spec.js | 121 ++++++++++++------ .../test/reporter/json_file_reporter_spec.js | 6 +- modules/benchpress/test/sampler_spec.js | 8 +- .../webdriver/ios_driver_extension_spec.js | 32 +---- 16 files changed, 214 insertions(+), 147 deletions(-) rename modules/benchpress/src/{sample_options.js => common_options.js} (73%) diff --git a/modules/benchpress/benchpress.es6 b/modules/benchpress/benchpress.es6 index 6db01a4c9f..7b1f0f23f2 100644 --- a/modules/benchpress/benchpress.es6 +++ b/modules/benchpress/benchpress.es6 @@ -1,14 +1,18 @@ import { bind } from 'angular2/di'; -import { JsonFileReporter } from './common'; +import { Options } from './common'; export * from './common'; export { SeleniumWebDriverAdapter } from './src/webdriver/selenium_webdriver_adapter'; var fs = require('fs'); -// Note: Can't do the `require` call in a facade as it can't be loaded into the browser! -JsonFileReporter.BINDINGS.push( - bind(JsonFileReporter.WRITE_FILE).toValue(writeFile) +// TODO(tbosch): right now we bind the `writeFile` method +// in benchpres/benchpress.es6. This does not work for Dart, +// find another way... +// Note: Can't do the `require` call in a facade as it can't be loaded into the browser +// for our unit tests via karma. +Options.DEFAULT_BINDINGS.push( + bind(Options.WRITE_FILE).toValue(writeFile) ); function writeFile(filename, content) { diff --git a/modules/benchpress/common.js b/modules/benchpress/common.js index a4df855c08..0da19f38f2 100644 --- a/modules/benchpress/common.js +++ b/modules/benchpress/common.js @@ -2,7 +2,7 @@ export { Sampler, SampleState } from './src/sampler'; export { Metric } from './src/metric'; export { Validator } from './src/validator'; export { Reporter } from './src/reporter'; -export { WebDriverExtension } from './src/web_driver_extension'; +export { WebDriverExtension, PerfLogFeatures } from './src/web_driver_extension'; export { WebDriverAdapter } from './src/web_driver_adapter'; export { SizeValidator } from './src/validator/size_validator'; export { RegressionSlopeValidator } from './src/validator/regression_slope_validator'; @@ -13,7 +13,7 @@ export { PerflogMetric } from './src/metric/perflog_metric'; export { ChromeDriverExtension } from './src/webdriver/chrome_driver_extension'; export { IOsDriverExtension } from './src/webdriver/ios_driver_extension'; export { Runner } from './src/runner'; -export { Options } from './src/sample_options'; +export { Options } from './src/common_options'; export { MeasureValues } from './src/measure_values'; export { MultiMetric } from './src/metric/multi_metric'; export { MultiReporter } from './src/reporter/multi_reporter'; diff --git a/modules/benchpress/src/sample_options.js b/modules/benchpress/src/common_options.js similarity index 73% rename from modules/benchpress/src/sample_options.js rename to modules/benchpress/src/common_options.js index b2c9801ae9..8af9df3825 100644 --- a/modules/benchpress/src/sample_options.js +++ b/modules/benchpress/src/common_options.js @@ -1,6 +1,8 @@ import { bind, OpaqueToken } from 'angular2/di'; +import { DateWrapper } from 'angular2/src/facade/lang'; export class Options { + static get DEFAULT_BINDINGS() { return _DEFAULT_BINDINGS; } // TODO(tbosch): use static initializer when our transpiler supports it static get SAMPLE_ID() { return _SAMPLE_ID; } // TODO(tbosch): use static initializer when our transpiler supports it @@ -23,6 +25,10 @@ export class Options { * Used for micro benchmarks. **/ static get MICRO_ITERATIONS() { return _MICRO_ITERATIONS; } + // TODO(tbosch): use static initializer when our transpiler supports it + static get NOW() { return _NOW; } + // TODO(tbosch): use static values when our transpiler supports them + static get WRITE_FILE() { return _WRITE_FILE; } } var _SAMPLE_ID = new OpaqueToken('Options.sampleId'); @@ -34,3 +40,13 @@ var _EXECUTE = new OpaqueToken('Options.execute'); var _CAPABILITIES = new OpaqueToken('Options.capabilities'); var _USER_AGENT = new OpaqueToken('Options.userAgent'); var _MICRO_ITERATIONS = new OpaqueToken('Options.microIterations'); +var _NOW = new OpaqueToken('Options.now'); +var _WRITE_FILE = new OpaqueToken('Options.writeFile'); + +var _DEFAULT_BINDINGS = [ + bind(_DEFAULT_DESCRIPTION).toValue({}), + bind(_SAMPLE_DESCRIPTION).toValue({}), + bind(_FORCE_GC).toValue(false), + bind(_PREPARE).toValue(false), + bind(_NOW).toValue( () => DateWrapper.now() ) +]; \ No newline at end of file diff --git a/modules/benchpress/src/metric/perflog_metric.js b/modules/benchpress/src/metric/perflog_metric.js index aa4bce4788..867a675adb 100644 --- a/modules/benchpress/src/metric/perflog_metric.js +++ b/modules/benchpress/src/metric/perflog_metric.js @@ -3,9 +3,9 @@ import { isPresent, isBlank, int, BaseException, StringWrapper, Math } from 'ang import { ListWrapper, StringMap, StringMapWrapper } from 'angular2/src/facade/collection'; import { bind, OpaqueToken } from 'angular2/di'; -import { WebDriverExtension } from '../web_driver_extension'; +import { WebDriverExtension, PerfLogFeatures } from '../web_driver_extension'; import { Metric } from '../metric'; -import { Options } from '../sample_options'; +import { Options } from '../common_options'; /** * A metric that reads out the performance log @@ -21,6 +21,7 @@ export class PerflogMetric extends Metric { _measureCount:int; _setTimeout:Function; _microIterations:int; + _perfLogFeatures:PerfLogFeatures; /** * @param driverExtension @@ -35,19 +36,24 @@ export class PerflogMetric extends Metric { this._measureCount = 0; this._setTimeout = setTimeout; this._microIterations = microIterations; + this._perfLogFeatures = driverExtension.perfLogFeatures(); } describe():StringMap { var res = { - 'script': 'script execution time in ms', - 'render': 'render time in ms', - 'gcTime': 'gc time in ms', - 'gcAmount': 'gc amount in kbytes', - 'majorGcTime': 'time of major gcs in ms', - 'majorGcAmount': 'amount of major gcs in kbytes' + 'scriptTime': 'script execution time in ms, including gc and render', + 'pureScriptTime': 'script execution time in ms, without gc nor render' }; + if (this._perfLogFeatures.render) { + res['renderTime'] = 'render time in and ouside of script in ms'; + } + if (this._perfLogFeatures.gc) { + res['gcTime'] = 'gc time in and ouside of script in ms'; + res['gcAmount'] = 'gc amount in kbytes'; + res['majorGcTime'] = 'time of major gcs in ms'; + } if (this._microIterations > 0) { - res['scriptMicroAvg'] = 'average script time for a micro iteration'; + res['microScriptTimeAvg'] = 'average script time for a micro iteration'; } return res; } @@ -120,17 +126,22 @@ export class PerflogMetric extends Metric { _aggregateEvents(events, markName) { var result = { - 'script': 0, - 'render': 0, - 'gcTime': 0, - 'gcAmount': 0, - 'majorGcTime': 0, - 'majorGcAmount': 0 + 'scriptTime': 0, + 'pureScriptTime': 0 }; + if (this._perfLogFeatures.gc) { + result['gcTime'] = 0; + result['majorGcTime'] = 0; + result['gcAmount'] = 0; + } + if (this._perfLogFeatures.render) { + result['renderTime'] = 0; + } var markStartEvent = null; var markEndEvent = null; var gcTimeInScript = 0; + var renderTimeInScript = 0; var intervalStarts = {}; events.forEach( (event) => { @@ -149,26 +160,30 @@ export class PerflogMetric extends Metric { var duration = event['ts'] - startEvent['ts']; intervalStarts[name] = null; if (StringWrapper.equals(name, 'gc')) { - var amount = (startEvent['args']['usedHeapSize'] - event['args']['usedHeapSize']) / 1000; result['gcTime'] += duration; + var amount = (startEvent['args']['usedHeapSize'] - event['args']['usedHeapSize']) / 1000; result['gcAmount'] += amount; var majorGc = event['args']['majorGc']; if (isPresent(majorGc) && majorGc) { result['majorGcTime'] += duration; - result['majorGcAmount'] += amount; } if (isPresent(intervalStarts['script'])) { gcTimeInScript += duration; } - } else if (StringWrapper.equals(name, 'script') || StringWrapper.equals(name, 'render')) { - result[name] += duration; + } else if (StringWrapper.equals(name, 'render')) { + result['renderTime'] += duration; + if (isPresent(intervalStarts['script'])) { + renderTimeInScript += duration; + } + } else if (StringWrapper.equals(name, 'script')) { + result['scriptTime'] += duration; } } } }); - result['script'] -= gcTimeInScript; + result['pureScriptTime'] = result['scriptTime'] - gcTimeInScript - renderTimeInScript; if (this._microIterations > 0) { - result['scriptMicroAvg'] = result['script'] / this._microIterations; + result['microScriptTimeAvg'] = result['scriptTime'] / this._microIterations; } return isPresent(markStartEvent) && isPresent(markEndEvent) ? result : null; } @@ -183,7 +198,8 @@ var _MARK_NAME_PREFIX = 'benchpress'; var _SET_TIMEOUT = new OpaqueToken('PerflogMetric.setTimeout'); var _BINDINGS = [ bind(PerflogMetric).toFactory( - (driverExtension, setTimeout, microIterations) => new PerflogMetric(driverExtension, setTimeout, microIterations), + (driverExtension, setTimeout, microIterations) => + new PerflogMetric(driverExtension, setTimeout, microIterations), [WebDriverExtension, _SET_TIMEOUT, Options.MICRO_ITERATIONS] ), bind(_SET_TIMEOUT).toValue( (fn, millis) => PromiseWrapper.setTimeout(fn, millis) ), diff --git a/modules/benchpress/src/reporter/json_file_reporter.js b/modules/benchpress/src/reporter/json_file_reporter.js index f431dea2a9..d1ccf50bb1 100644 --- a/modules/benchpress/src/reporter/json_file_reporter.js +++ b/modules/benchpress/src/reporter/json_file_reporter.js @@ -7,16 +7,12 @@ import { bind, OpaqueToken } from 'angular2/di'; import { Reporter } from '../reporter'; import { SampleDescription } from '../sample_description'; import { MeasureValues } from '../measure_values'; +import { Options } from '../common_options'; /** * A reporter that writes results into a json file. - * TODO(tbosch): right now we bind the `writeFile` method - * in benchpres/benchpress.es6. This does not work for Dart, - * find another way... */ export class JsonFileReporter extends Reporter { - // TODO(tbosch): use static values when our transpiler supports them - static get WRITE_FILE() { return _WRITE_FILE; } // TODO(tbosch): use static values when our transpiler supports them static get PATH() { return _PATH; } static get BINDINGS() { return _BINDINGS; } @@ -24,12 +20,14 @@ export class JsonFileReporter extends Reporter { _writeFile:Function; _path:string; _description:SampleDescription; + _now:Function; - constructor(sampleDescription, path, writeFile) { + constructor(sampleDescription, path, writeFile, now) { super(); this._description = sampleDescription; this._path = path; this._writeFile = writeFile; + this._now = now; } reportMeasureValues(measureValues:MeasureValues):Promise { @@ -42,17 +40,16 @@ export class JsonFileReporter extends Reporter { 'completeSample': completeSample, 'validSample': validSample }); - var filePath = `${this._path}/${this._description.id}_${DateWrapper.toMillis(DateWrapper.now())}.json`; + var filePath = `${this._path}/${this._description.id}_${DateWrapper.toMillis(this._now())}.json`; return this._writeFile(filePath, content); } } -var _WRITE_FILE = new OpaqueToken('JsonFileReporter.writeFile'); var _PATH = new OpaqueToken('JsonFileReporter.path'); var _BINDINGS = [ bind(JsonFileReporter).toFactory( - (sampleDescription, path, writeFile) => new JsonFileReporter(sampleDescription, path, writeFile), - [SampleDescription, _PATH, _WRITE_FILE] + (sampleDescription, path, writeFile, now) => new JsonFileReporter(sampleDescription, path, writeFile, now), + [SampleDescription, _PATH, Options.WRITE_FILE, Options.NOW] ), bind(_PATH).toValue('.') ]; diff --git a/modules/benchpress/src/runner.js b/modules/benchpress/src/runner.js index 35a6c3d892..6425e2469d 100644 --- a/modules/benchpress/src/runner.js +++ b/modules/benchpress/src/runner.js @@ -18,7 +18,7 @@ import { SampleDescription } from './sample_description'; import { WebDriverAdapter } from './web_driver_adapter'; import { Reporter } from './reporter'; import { Metric } from './metric'; -import { Options } from './sample_options'; +import { Options } from './common_options'; /** * The Runner is the main entry point for executing a sample run. @@ -56,6 +56,7 @@ export class Runner { } var _DEFAULT_BINDINGS = [ + Options.DEFAULT_BINDINGS, Sampler.BINDINGS, ConsoleReporter.BINDINGS, RegressionSlopeValidator.BINDINGS, diff --git a/modules/benchpress/src/sample_description.js b/modules/benchpress/src/sample_description.js index 773ecda8ae..56949d36ce 100644 --- a/modules/benchpress/src/sample_description.js +++ b/modules/benchpress/src/sample_description.js @@ -2,7 +2,7 @@ import { StringMapWrapper, ListWrapper, StringMap } from 'angular2/src/facade/co import { bind, OpaqueToken } from 'angular2/di'; import { Validator } from './validator'; import { Metric } from './metric'; -import { Options } from './sample_options'; +import { Options } from './common_options'; /** * SampleDescription merges all available descriptions about a sample @@ -47,7 +47,5 @@ var _BINDINGS = [ Metric, Options.SAMPLE_ID, Options.FORCE_GC, Options.USER_AGENT, Validator, Options.DEFAULT_DESCRIPTION, Options.SAMPLE_DESCRIPTION ] - ), - bind(Options.DEFAULT_DESCRIPTION).toValue({}), - bind(Options.SAMPLE_DESCRIPTION).toValue({}) + ) ]; diff --git a/modules/benchpress/src/sampler.js b/modules/benchpress/src/sampler.js index d43fd304ae..5f7a0d27dc 100644 --- a/modules/benchpress/src/sampler.js +++ b/modules/benchpress/src/sampler.js @@ -9,7 +9,7 @@ import { Reporter } from './reporter'; import { WebDriverExtension } from './web_driver_extension'; import { WebDriverAdapter } from './web_driver_adapter'; -import { Options } from './sample_options'; +import { Options } from './common_options'; import { MeasureValues} from './measure_values'; /** @@ -23,8 +23,6 @@ import { MeasureValues} from './measure_values'; export class Sampler { // TODO(tbosch): use static values when our transpiler supports them static get BINDINGS() { return _BINDINGS; } - // TODO(tbosch): use static values when our transpiler supports them - static get TIME() { return _TIME; } _driver:WebDriverAdapter; _driverExtension:WebDriverExtension; @@ -34,14 +32,14 @@ export class Sampler { _forceGc:boolean; _prepare:Function; _execute:Function; - _time:Function; + _now:Function; constructor({ - driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, time + driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, now }:{ driver: WebDriverAdapter, driverExtension: WebDriverExtension, metric: Metric, reporter: Reporter, - validator: Validator, prepare: Function, execute: Function, time: Function + validator: Validator, prepare: Function, execute: Function, now: Function }={}) { this._driver = driver; this._driverExtension = driverExtension; @@ -51,7 +49,7 @@ export class Sampler { this._forceGc = forceGc; this._prepare = prepare; this._execute = execute; - this._time = time; + this._now = now; } sample():Promise { @@ -96,7 +94,7 @@ export class Sampler { } _report(state:SampleState, metricValues:StringMap):Promise { - var measureValues = new MeasureValues(state.completeSample.length, this._time(), metricValues); + var measureValues = new MeasureValues(state.completeSample.length, this._now(), metricValues); var completeSample = ListWrapper.concat(state.completeSample, [measureValues]); var validSample = this._validator.validate(completeSample); var resultPromise = this._reporter.reportMeasureValues(measureValues); @@ -118,11 +116,9 @@ export class SampleState { } } -var _TIME = new OpaqueToken('Sampler.time'); - var _BINDINGS = [ bind(Sampler).toFactory( - (driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, time) => new Sampler({ + (driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, now) => new Sampler({ driver: driver, driverExtension: driverExtension, reporter: reporter, @@ -134,14 +130,11 @@ var _BINDINGS = [ // special null object, which is expensive. prepare: prepare !== false ? prepare : null, execute: execute, - time: time + now: now }), [ WebDriverAdapter, WebDriverExtension, Metric, Reporter, Validator, - Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, _TIME + Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, Options.NOW ] - ), - bind(Options.FORCE_GC).toValue(false), - bind(Options.PREPARE).toValue(false), - bind(_TIME).toValue( () => DateWrapper.now() ) + ) ]; diff --git a/modules/benchpress/src/validator/regression_slope_validator.js b/modules/benchpress/src/validator/regression_slope_validator.js index 3143a896f5..6960b839d4 100644 --- a/modules/benchpress/src/validator/regression_slope_validator.js +++ b/modules/benchpress/src/validator/regression_slope_validator.js @@ -65,5 +65,5 @@ var _BINDINGS = [ [_SAMPLE_SIZE, _METRIC] ), bind(_SAMPLE_SIZE).toValue(10), - bind(_METRIC).toValue('script') + bind(_METRIC).toValue('scriptTime') ]; diff --git a/modules/benchpress/src/web_driver_extension.js b/modules/benchpress/src/web_driver_extension.js index b2849c6dfc..9d70a3c2a1 100644 --- a/modules/benchpress/src/web_driver_extension.js +++ b/modules/benchpress/src/web_driver_extension.js @@ -1,10 +1,10 @@ import { bind, Injector, OpaqueToken } from 'angular2/di'; -import { BaseException, ABSTRACT, isBlank } from 'angular2/src/facade/lang'; +import { BaseException, ABSTRACT, isBlank, isPresent } from 'angular2/src/facade/lang'; import { Promise, PromiseWrapper } from 'angular2/src/facade/async'; import { List, ListWrapper, StringMap } from 'angular2/src/facade/collection'; -import { Options } from './sample_options'; +import { Options } from './common_options'; /** * A WebDriverExtension implements extended commands of the webdriver protocol @@ -64,9 +64,23 @@ export class WebDriverExtension { throw new BaseException('NYI'); } + perfLogFeatures():PerfLogFeatures { + throw new BaseException('NYI'); + } + supports(capabilities:StringMap):boolean { return true; } } +export class PerfLogFeatures { + render:boolean; + gc:boolean; + + constructor({render, gc} = {}) { + this.render = isPresent(render) && render; + this.gc = isPresent(gc) && gc; + } +} + var _CHILDREN = new OpaqueToken('WebDriverExtension.children'); diff --git a/modules/benchpress/src/webdriver/chrome_driver_extension.js b/modules/benchpress/src/webdriver/chrome_driver_extension.js index aa3ab0ebb0..19956f16d1 100644 --- a/modules/benchpress/src/webdriver/chrome_driver_extension.js +++ b/modules/benchpress/src/webdriver/chrome_driver_extension.js @@ -4,7 +4,7 @@ import { Json, isPresent, isBlank, RegExpWrapper, StringWrapper, BaseException, NumberWrapper } from 'angular2/src/facade/lang'; -import { WebDriverExtension } from '../web_driver_extension'; +import { WebDriverExtension, PerfLogFeatures } from '../web_driver_extension'; import { WebDriverAdapter } from '../web_driver_adapter'; import { Promise } from 'angular2/src/facade/async'; @@ -111,6 +111,13 @@ export class ChromeDriverExtension extends WebDriverExtension { return normalizedEvents; } + perfLogFeatures():PerfLogFeatures { + return new PerfLogFeatures({ + render: true, + gc: true + }); + } + supports(capabilities:StringMap):boolean { return StringWrapper.equals(capabilities['browserName'].toLowerCase(), 'chrome'); } diff --git a/modules/benchpress/src/webdriver/ios_driver_extension.js b/modules/benchpress/src/webdriver/ios_driver_extension.js index fdd2f5fac2..80d92d23a7 100644 --- a/modules/benchpress/src/webdriver/ios_driver_extension.js +++ b/modules/benchpress/src/webdriver/ios_driver_extension.js @@ -1,10 +1,10 @@ import { bind } from 'angular2/di'; import { ListWrapper, StringMap } from 'angular2/src/facade/collection'; import { - Json, isPresent, isBlank, RegExpWrapper, StringWrapper + Json, isPresent, isBlank, RegExpWrapper, StringWrapper, BaseException } from 'angular2/src/facade/lang'; -import { WebDriverExtension } from '../web_driver_extension'; +import { WebDriverExtension, PerfLogFeatures } from '../web_driver_extension'; import { WebDriverAdapter } from '../web_driver_adapter'; import { Promise } from 'angular2/src/facade/async'; @@ -21,7 +21,7 @@ export class IOsDriverExtension extends WebDriverExtension { } gc() { - return this._driver.executeScript('window.gc()'); + throw new BaseException('Force GC is not supported on iOS'); } timeBegin(name:string):Promise { @@ -81,14 +81,8 @@ export class IOsDriverExtension extends WebDriverExtension { StringWrapper.equals(type, 'CompositeLayers')) { ListWrapper.push(events, createStartEvent('render', startTime)); endEvent = createEndEvent('render', endTime); - } else if (StringWrapper.equals(type, 'GCEvent')) { - ListWrapper.push(events, createStartEvent('gc', startTime, { - 'usedHeapSize': 0 - })); - endEvent = createEndEvent('gc', endTime, { - 'usedHeapSize': -data['usedHeapSizeDelta'] - }); } + // Note: ios used to support GCEvent up until iOS 6 :-( if (isPresent(record['children'])) { this._convertPerfRecordsToEvents(record['children'], events); } @@ -99,6 +93,12 @@ export class IOsDriverExtension extends WebDriverExtension { return events; } + perfLogFeatures():PerfLogFeatures { + return new PerfLogFeatures({ + render: true + }); + } + supports(capabilities:StringMap):boolean { return StringWrapper.equals(capabilities['browserName'].toLowerCase(), 'safari'); } diff --git a/modules/benchpress/test/metric/perflog_metric_spec.js b/modules/benchpress/test/metric/perflog_metric_spec.js index f385d858a8..e3786ffb38 100644 --- a/modules/benchpress/test/metric/perflog_metric_spec.js +++ b/modules/benchpress/test/metric/perflog_metric_spec.js @@ -13,9 +13,13 @@ import { import { List, ListWrapper } from 'angular2/src/facade/collection'; import { PromiseWrapper, Promise } from 'angular2/src/facade/async'; -import { isPresent } from 'angular2/src/facade/lang'; +import { isPresent, isBlank } from 'angular2/src/facade/lang'; -import { Metric, PerflogMetric, WebDriverExtension, bind, Injector, Options } from 'benchpress/common'; +import { + Metric, PerflogMetric, WebDriverExtension, + PerfLogFeatures, + bind, Injector, Options +} from 'benchpress/common'; import { TraceEventFactory } from '../trace_event_factory'; @@ -23,15 +27,19 @@ export function main() { var commandLog; var eventFactory = new TraceEventFactory('timeline', 'pid0'); - function createMetric(perfLogs, microIterations = 0) { + function createMetric(perfLogs, microIterations = 0, perfLogFeatures = null) { commandLog = []; + if (isBlank(perfLogFeatures)) { + perfLogFeatures = new PerfLogFeatures({render: true, gc: true}); + } var bindings = [ + Options.DEFAULT_BINDINGS, PerflogMetric.BINDINGS, bind(PerflogMetric.SET_TIMEOUT).toValue( (fn, millis) => { ListWrapper.push(commandLog, ['setTimeout', millis]); fn(); }), - bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog)), + bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog, perfLogFeatures)), bind(Options.MICRO_ITERATIONS).toValue(microIterations) ]; return new Injector(bindings).get(PerflogMetric); @@ -39,8 +47,29 @@ export function main() { describe('perflog metric', () => { - it('should describe itself', () => { - expect(createMetric([[]]).describe()['script']).toBe('script execution time in ms'); + it('should describe itself based on the perfLogFeatrues', () => { + expect(createMetric([[]], 0, new PerfLogFeatures()).describe()).toEqual({ + 'scriptTime': 'script execution time in ms, including gc and render', + 'pureScriptTime': 'script execution time in ms, without gc nor render' + }); + + expect(createMetric([[]], 0, new PerfLogFeatures({ + render: true, + gc: false + })).describe()).toEqual({ + 'scriptTime': 'script execution time in ms, including gc and render', + 'pureScriptTime': 'script execution time in ms, without gc nor render', + 'renderTime': 'render time in and ouside of script in ms', + }); + + expect(createMetric([[]]).describe()).toEqual({ + 'scriptTime': 'script execution time in ms, including gc and render', + 'pureScriptTime': 'script execution time in ms, without gc nor render', + 'renderTime': 'render time in and ouside of script in ms', + 'gcTime': 'gc time in and ouside of script in ms', + 'gcAmount': 'gc amount in kbytes', + 'majorGcTime': 'time of major gcs in ms' + }); }); describe('beginMeasure', () => { @@ -76,7 +105,7 @@ export function main() { ['timeEnd', 'benchpress0', null], 'readPerfLog' ]); - expect(data['script']).toBe(2); + expect(data['scriptTime']).toBe(2); async.done(); }); @@ -128,7 +157,7 @@ export function main() { [ 'setTimeout', 100 ], 'readPerfLog' ]); - expect(data['script']).toBe(3); + expect(data['scriptTime']).toBe(3); async.done(); }); @@ -144,7 +173,7 @@ export function main() { metric.beginMeasure() .then( (_) => metric.endMeasure(true) ) .then( (data) => { - expect(data['script']).toBe(0); + expect(data['scriptTime']).toBe(0); return metric.endMeasure(true) }) .then( (data) => { @@ -155,7 +184,7 @@ export function main() { ['timeEnd', 'benchpress1', 'benchpress2'], 'readPerfLog' ]); - expect(data['script']).toBe(3); + expect(data['scriptTime']).toBe(3); async.done(); }); @@ -179,7 +208,7 @@ export function main() { eventFactory.start('script', 0), eventFactory.end('script', 5) ]).then((data) => { - expect(data['script']).toBe(5); + expect(data['scriptTime']).toBe(5); async.done(); }); })); @@ -191,7 +220,7 @@ export function main() { eventFactory.start('script', 10), eventFactory.end('script', 17) ]).then((data) => { - expect(data['script']).toBe(12); + expect(data['scriptTime']).toBe(12); async.done(); }); })); @@ -200,7 +229,7 @@ export function main() { aggregate([ eventFactory.end('script', 10) ]).then((data) => { - expect(data['script']).toBe(0); + expect(data['scriptTime']).toBe(0); async.done(); }); })); @@ -209,7 +238,7 @@ export function main() { aggregate([ eventFactory.start('script', 10) ]).then((data) => { - expect(data['script']).toBe(0); + expect(data['scriptTime']).toBe(0); async.done(); }); })); @@ -227,22 +256,30 @@ export function main() { metric.beginMeasure() .then( (_) => metric.endMeasure(false) ) .then((data) => { - expect(data['script']).toBe(5); + expect(data['scriptTime']).toBe(5); async.done(); }); })); - ['script', 'render'].forEach( (metricName) => { - it(`should support ${metricName} metric`, inject([AsyncTestCompleter], (async) => { - aggregate([ - eventFactory.start(metricName, 0), - eventFactory.end(metricName, 5) - ]).then((data) => { - expect(data[metricName]).toBe(5); - async.done(); - }); - })); - }); + it('should support scriptTime metric', inject([AsyncTestCompleter], (async) => { + aggregate([ + eventFactory.start('script', 0), + eventFactory.end('script', 5) + ]).then((data) => { + expect(data['scriptTime']).toBe(5); + async.done(); + }); + })); + + it('should support renderTime metric', inject([AsyncTestCompleter], (async) => { + aggregate([ + eventFactory.start('render', 0), + eventFactory.end('render', 5) + ]).then((data) => { + expect(data['renderTime']).toBe(5); + async.done(); + }); + })); it('should support gcTime/gcAmount metric', inject([AsyncTestCompleter], (async) => { aggregate([ @@ -252,55 +289,55 @@ export function main() { expect(data['gcTime']).toBe(5); expect(data['gcAmount']).toBe(1.5); expect(data['majorGcTime']).toBe(0); - expect(data['majorGcAmount']).toBe(0); async.done(); }); })); - it('should support majorGcTime/majorGcAmount metric', inject([AsyncTestCompleter], (async) => { + it('should support majorGcTime metric', inject([AsyncTestCompleter], (async) => { aggregate([ eventFactory.start('gc', 0, {'usedHeapSize': 2500}), eventFactory.end('gc', 5, {'usedHeapSize': 1000, 'majorGc': true}) ]).then((data) => { expect(data['gcTime']).toBe(5); - expect(data['gcAmount']).toBe(1.5); expect(data['majorGcTime']).toBe(5); - expect(data['majorGcAmount']).toBe(1.5); async.done(); }); })); - it('should subtract gcTime in script from script time', inject([AsyncTestCompleter], (async) => { + it('should support pureScriptTime = scriptTime-gcTime-renderTime', inject([AsyncTestCompleter], (async) => { aggregate([ eventFactory.start('script', 0), eventFactory.start('gc', 1, {'usedHeapSize': 1000}), eventFactory.end('gc', 4, {'usedHeapSize': 0}), - eventFactory.end('script', 5) + eventFactory.start('render', 4), + eventFactory.end('render', 5), + eventFactory.end('script', 6) ]).then((data) => { - expect(data['script']).toBe(2); + expect(data['scriptTime']).toBe(6); + expect(data['pureScriptTime']).toBe(2); async.done(); }); })); describe('microIterations', () => { - it('should not report scriptMicroAvg if microIterations = 0', inject([AsyncTestCompleter], (async) => { + it('should not report microScriptTimeAvg if microIterations = 0', inject([AsyncTestCompleter], (async) => { aggregate([ eventFactory.start('script', 0), eventFactory.end('script', 5) ], 0).then((data) => { - expect(isPresent(data['scriptMicroAvg'])).toBe(false); + expect(isPresent(data['microScriptTimeAvg'])).toBe(false); async.done(); }); })); - it('should report scriptMicroAvg', inject([AsyncTestCompleter], (async) => { + it('should report microScriptTimeAvg', inject([AsyncTestCompleter], (async) => { aggregate([ eventFactory.start('script', 0), eventFactory.end('script', 5) ], 4).then((data) => { - expect(data['script']).toBe(5); - expect(data['scriptMicroAvg']).toBe(5/4); + expect(data['scriptTime']).toBe(5); + expect(data['microScriptTimeAvg']).toBe(5/4); async.done(); }); })); @@ -315,10 +352,12 @@ export function main() { class MockDriverExtension extends WebDriverExtension { _perfLogs:List; _commandLog:List; - constructor(perfLogs, commandLog) { + _perfLogFeatures:PerfLogFeatures; + constructor(perfLogs, commandLog, perfLogFeatures) { super(); this._perfLogs = perfLogs; this._commandLog = commandLog; + this._perfLogFeatures = perfLogFeatures; } timeBegin(name):Promise { @@ -331,6 +370,10 @@ class MockDriverExtension extends WebDriverExtension { return PromiseWrapper.resolve(null); } + perfLogFeatures():PerfLogFeatures { + return this._perfLogFeatures; + } + readPerfLog():Promise { ListWrapper.push(this._commandLog, 'readPerfLog'); if (this._perfLogs.length > 0) { diff --git a/modules/benchpress/test/reporter/json_file_reporter_spec.js b/modules/benchpress/test/reporter/json_file_reporter_spec.js index 8f51c2670d..16f4a44e01 100644 --- a/modules/benchpress/test/reporter/json_file_reporter_spec.js +++ b/modules/benchpress/test/reporter/json_file_reporter_spec.js @@ -17,7 +17,8 @@ import { PromiseWrapper } from 'angular2/src/facade/async'; import { bind, Injector, SampleDescription, - MeasureValues + MeasureValues, + Options } from 'benchpress/common'; @@ -32,7 +33,8 @@ export function main() { JsonFileReporter.BINDINGS, bind(SampleDescription).toValue(new SampleDescription(sampleId, descriptions, metrics)), bind(JsonFileReporter.PATH).toValue(path), - bind(JsonFileReporter.WRITE_FILE).toValue((filename, content) => { + bind(Options.NOW).toValue( () => DateWrapper.fromMillis(1234) ), + bind(Options.WRITE_FILE).toValue((filename, content) => { loggedFile = { 'filename': filename, 'content': content diff --git a/modules/benchpress/test/sampler_spec.js b/modules/benchpress/test/sampler_spec.js index 64e110bde0..d6da57f7f9 100644 --- a/modules/benchpress/test/sampler_spec.js +++ b/modules/benchpress/test/sampler_spec.js @@ -50,15 +50,17 @@ export function main() { if (isBlank(driverExtension)) { driverExtension = new MockDriverExtension([]); } - var bindings = ListWrapper.concat(Sampler.BINDINGS, [ + var bindings = [ + Options.DEFAULT_BINDINGS, + Sampler.BINDINGS, bind(Metric).toValue(metric), bind(Reporter).toValue(reporter), bind(WebDriverAdapter).toValue(driver), bind(WebDriverExtension).toValue(driverExtension), bind(Options.EXECUTE).toValue(execute), bind(Validator).toValue(validator), - bind(Sampler.TIME).toValue( () => DateWrapper.fromMillis(time++) ) - ]); + bind(Options.NOW).toValue( () => DateWrapper.fromMillis(time++) ) + ]; if (isPresent(prepare)) { ListWrapper.push(bindings, bind(Options.PREPARE).toValue(prepare)); } diff --git a/modules/benchpress/test/webdriver/ios_driver_extension_spec.js b/modules/benchpress/test/webdriver/ios_driver_extension_spec.js index 4e93eeaed3..0d43707a0e 100644 --- a/modules/benchpress/test/webdriver/ios_driver_extension_spec.js +++ b/modules/benchpress/test/webdriver/ios_driver_extension_spec.js @@ -41,12 +41,9 @@ export function main() { return extension; } - it('should force gc via window.gc()', inject([AsyncTestCompleter], (async) => { - createExtension().gc().then( (_) => { - expect(log).toEqual([['executeScript', 'window.gc()']]); - async.done(); - }); - })); + it('should throw on forcing gc', () => { + expect( () => createExtension().gc() ).toThrowError('Force GC is not supported on iOS'); + }); it('should mark the timeline via console.time()', inject([AsyncTestCompleter], (async) => { createExtension().timeBegin('someName').then( (_) => { @@ -124,18 +121,6 @@ export function main() { }); })); - it('should report gc', inject([AsyncTestCompleter], (async) => { - createExtension([ - gcRecord(1, 3, 21) - ]).readPerfLog().then( (events) => { - expect(events).toEqual([ - normEvents.start('gc', 1, {'usedHeapSize': 0}), - normEvents.end('gc', 3, {'usedHeapSize': -21}), - ]); - async.done(); - }); - })); - ['RecalculateStyles', 'Layout', 'UpdateLayerTree', 'Paint', 'Rasterize', 'CompositeLayers'].forEach( (recordType) => { it(`should report ${recordType}`, inject([AsyncTestCompleter], (async) => { createExtension([ @@ -224,17 +209,6 @@ function internalScriptRecord(startTime, endTime) { }; } -function gcRecord(startTime, endTime, gcAmount) { - return { - 'type': 'GCEvent', - 'startTime': startTime, - 'endTime': endTime, - 'data': { - 'usedHeapSizeDelta': gcAmount - } - }; -} - class MockDriverAdapter extends WebDriverAdapter { _log:List; _perfRecords:List;