diff --git a/modules/angular2/src/test_lib/perf_util.es6 b/modules/angular2/src/test_lib/perf_util.es6 index 9361642af0..226414f425 100644 --- a/modules/angular2/src/test_lib/perf_util.es6 +++ b/modules/angular2/src/test_lib/perf_util.es6 @@ -23,7 +23,14 @@ function runBenchmark(config) { return getScaleFactor(browser.params.benchmark.scaling).then(function(scaleFactor) { var description = {}; var urlParams = []; - config.params.forEach(function(param) { + var microIterations = config.microIterations || 0; + var params = config.params || []; + if (microIterations) { + params = params.concat([{ + name: 'iterations', value: microIterations, scale: 'linear' + }]); + } + params.forEach(function(param) { var name = param.name; var value = applyScaleFactor(param.value, scaleFactor, param.scale); urlParams.push(name + '=' + value); @@ -35,6 +42,7 @@ function runBenchmark(config) { id: config.id, execute: config.work, prepare: config.prepare, + microIterations: microIterations, bindings: [ benchpress.bind(benchpress.Options.SAMPLE_DESCRIPTION).toValue(description) ] diff --git a/modules/benchmarks/e2e_test/change_detection_perf.es6 b/modules/benchmarks/e2e_test/change_detection_perf.es6 index 484dda569b..99eb82645f 100644 --- a/modules/benchmarks/e2e_test/change_detection_perf.es6 +++ b/modules/benchmarks/e2e_test/change_detection_perf.es6 @@ -12,8 +12,9 @@ describe('ng2 change detection benchmark', function () { buttons: ['#ng2ChangeDetectionDynamic'], id: 'ng2.changeDetection.dynamic', params: [{ - name: 'numberOfChecks', value: 900000, scale: 'linear' - }] + name: 'numberOfChecks', value: 900000 + }], + microIterations: 20 }).then(done, done.fail); }); @@ -23,8 +24,9 @@ describe('ng2 change detection benchmark', function () { buttons: ['#ng2ChangeDetectionJit'], id: 'ng2.changeDetection.jit', params: [{ - name: 'numberOfChecks', value: 900000, scale: 'linear' - }] + name: 'numberOfChecks', value: 900000 + }], + microIterations: 20 }).then(done, done.fail); }); @@ -34,8 +36,9 @@ describe('ng2 change detection benchmark', function () { buttons: ['#baselineChangeDetection'], id: 'baseline.changeDetection', params: [{ - name: 'numberOfChecks', value: 900000, scale: 'linear' - }] + name: 'numberOfChecks', value: 900000 + }], + microIterations: 20 }).then(done, done.fail); }); diff --git a/modules/benchmarks/e2e_test/di_perf.es6 b/modules/benchmarks/e2e_test/di_perf.es6 index 1158de8eb2..d987a20da0 100644 --- a/modules/benchmarks/e2e_test/di_perf.es6 +++ b/modules/benchmarks/e2e_test/di_perf.es6 @@ -13,7 +13,8 @@ describe('ng2 di benchmark', function () { id: 'ng2.di.getByToken', params: [{ name: 'iterations', value: 20000, scale: 'linear' - }] + }], + microIterations: 20000 }).then(done, done.fail); }); @@ -22,9 +23,7 @@ describe('ng2 di benchmark', function () { url: URL, buttons: ['#getByKey'], id: 'ng2.di.getByKey', - params: [{ - name: 'iterations', value: 20000, scale: 'linear' - }] + microIterations: 20000 }).then(done, done.fail); }); @@ -33,9 +32,7 @@ describe('ng2 di benchmark', function () { url: URL, buttons: ['#getChild'], id: 'ng2.di.getChild', - params: [{ - name: 'iterations', value: 20000, scale: 'linear' - }] + microIterations: 20000 }).then(done, done.fail); }); @@ -44,9 +41,7 @@ describe('ng2 di benchmark', function () { url: URL, buttons: ['#instantiate'], id: 'ng2.di.instantiate', - params: [{ - name: 'iterations', value: 10000, scale: 'linear' - }] + microIterations: 10000 }).then(done, done.fail); }); diff --git a/modules/benchmarks/e2e_test/element_injector_perf.es6 b/modules/benchmarks/e2e_test/element_injector_perf.es6 index 2e93597205..f92b769a8b 100644 --- a/modules/benchmarks/e2e_test/element_injector_perf.es6 +++ b/modules/benchmarks/e2e_test/element_injector_perf.es6 @@ -11,9 +11,7 @@ describe('ng2 element injector benchmark', function () { url: URL, buttons: ['#instantiate'], id: 'ng2.elementInjector.instantiate', - params: [{ - name: 'iterations', value: 20000, scale: 'linear' - }] + microIterations: 20000 }).then(done, done.fail); }); @@ -22,9 +20,7 @@ describe('ng2 element injector benchmark', function () { url: URL, buttons: ['#instantiateDirectives'], id: 'ng2.elementInjector.instantiateDirectives', - params: [{ - name: 'iterations', value: 20000, scale: 'linear' - }] + microIterations: 20000 }).then(done, done.fail); }); diff --git a/modules/benchmarks/src/change_detection/change_detection_benchmark.html b/modules/benchmarks/src/change_detection/change_detection_benchmark.html index 6875f1ad84..ea5618c07a 100644 --- a/modules/benchmarks/src/change_detection/change_detection_benchmark.html +++ b/modules/benchmarks/src/change_detection/change_detection_benchmark.html @@ -5,6 +5,8 @@

Params

Iterations: + + Number of checks:
diff --git a/modules/benchmarks/src/change_detection/change_detection_benchmark.js b/modules/benchmarks/src/change_detection/change_detection_benchmark.js index 65b19e785f..ff1830425f 100644 --- a/modules/benchmarks/src/change_detection/change_detection_benchmark.js +++ b/modules/benchmarks/src/change_detection/change_detection_benchmark.js @@ -138,9 +138,9 @@ function setUpChangeDetection(changeDetection:ChangeDetection, iterations) { export function main () { BrowserDomAdapter.makeCurrent(); var numberOfChecks = getIntParameter('numberOfChecks'); + var numberOfRuns = getIntParameter('iterations'); var numberOfChecksPerDetector = 10; - var numberOfRuns = 20; var numberOfDetectors = numberOfChecks / numberOfChecksPerDetector / numberOfRuns; setUpReflector(); diff --git a/modules/benchpress/src/metric/perflog_metric.js b/modules/benchpress/src/metric/perflog_metric.js index 9cc401d9c5..4345509455 100644 --- a/modules/benchpress/src/metric/perflog_metric.js +++ b/modules/benchpress/src/metric/perflog_metric.js @@ -5,6 +5,7 @@ import { bind, OpaqueToken } from 'angular2/di'; import { WebDriverExtension } from '../web_driver_extension'; import { Metric } from '../metric'; +import { Options } from '../sample_options'; /** * A metric that reads out the performance log @@ -19,17 +20,25 @@ export class PerflogMetric extends Metric { _remainingEvents:List; _measureCount:int; _setTimeout:Function; + _microIterations:int; - constructor(driverExtension:WebDriverExtension, setTimeout:Function) { + /** + * @param driverExtension + * @param setTimeout + * @param microIterations Number of iterations that run inside the browser by user code. + * Used for micro benchmarks. + **/ + constructor(driverExtension:WebDriverExtension, setTimeout:Function, microIterations:int) { super(); this._driverExtension = driverExtension; this._remainingEvents = []; this._measureCount = 0; this._setTimeout = setTimeout; + this._microIterations = microIterations; } describe():StringMap { - return { + var res = { 'script': 'script execution time in ms', 'render': 'render time in ms', 'gcTime': 'gc time in ms', @@ -37,6 +46,10 @@ export class PerflogMetric extends Metric { 'gcTimeInScript': 'gc time during script execution in ms', 'gcAmountInScript': 'gc amount during script execution in kbytes' }; + if (this._microIterations > 0) { + res['scriptMicroAvg'] = 'average script time for a micro iteration'; + } + return res; } beginMeasure():Promise { @@ -148,6 +161,9 @@ export class PerflogMetric extends Metric { } }); result['script'] -= result['gcTimeInScript']; + if (this._microIterations > 0) { + result['scriptMicroAvg'] = result['script'] / this._microIterations; + } return isPresent(markStartEvent) && isPresent(markEndEvent) ? result : null; } @@ -161,8 +177,9 @@ var _MARK_NAME_PREFIX = 'benchpress'; var _SET_TIMEOUT = new OpaqueToken('PerflogMetric.setTimeout'); var _BINDINGS = [ bind(PerflogMetric).toFactory( - (driverExtension, setTimeout) => new PerflogMetric(driverExtension, setTimeout), - [WebDriverExtension, _SET_TIMEOUT] + (driverExtension, setTimeout, microIterations) => new PerflogMetric(driverExtension, setTimeout, microIterations), + [WebDriverExtension, _SET_TIMEOUT, Options.MICRO_ITERATIONS] ), - bind(_SET_TIMEOUT).toValue( (fn, millis) => PromiseWrapper.setTimeout(fn, millis) ) + bind(_SET_TIMEOUT).toValue( (fn, millis) => PromiseWrapper.setTimeout(fn, millis) ), + bind(Options.MICRO_ITERATIONS).toValue(0) ]; diff --git a/modules/benchpress/src/runner.js b/modules/benchpress/src/runner.js index 527be7dcc7..35a6c3d892 100644 --- a/modules/benchpress/src/runner.js +++ b/modules/benchpress/src/runner.js @@ -34,7 +34,7 @@ export class Runner { this._defaultBindings = defaultBindings; } - sample({id, execute, prepare, bindings}):Promise { + sample({id, execute, prepare, microIterations, bindings}):Promise { var sampleBindings = [ _DEFAULT_BINDINGS, this._defaultBindings, @@ -44,6 +44,9 @@ export class Runner { if (isPresent(prepare)) { ListWrapper.push(sampleBindings, bind(Options.PREPARE).toValue(prepare)); } + if (isPresent(microIterations)) { + ListWrapper.push(sampleBindings, bind(Options.MICRO_ITERATIONS).toValue(microIterations)); + } if (isPresent(bindings)) { ListWrapper.push(sampleBindings, bindings); } diff --git a/modules/benchpress/src/sample_options.js b/modules/benchpress/src/sample_options.js index e403950137..b2c9801ae9 100644 --- a/modules/benchpress/src/sample_options.js +++ b/modules/benchpress/src/sample_options.js @@ -17,6 +17,12 @@ export class Options { static get CAPABILITIES() { return _CAPABILITIES; } // TODO(tbosch): use static initializer when our transpiler supports it static get USER_AGENT() { return _USER_AGENT; } + // TODO(tbosch): use static initializer when our transpiler supports it + /** + * Number of iterations that run inside the browser by user code. + * Used for micro benchmarks. + **/ + static get MICRO_ITERATIONS() { return _MICRO_ITERATIONS; } } var _SAMPLE_ID = new OpaqueToken('Options.sampleId'); @@ -27,3 +33,4 @@ var _PREPARE = new OpaqueToken('Options.prepare'); var _EXECUTE = new OpaqueToken('Options.execute'); var _CAPABILITIES = new OpaqueToken('Options.capabilities'); var _USER_AGENT = new OpaqueToken('Options.userAgent'); +var _MICRO_ITERATIONS = new OpaqueToken('Options.microIterations'); diff --git a/modules/benchpress/src/sampler.js b/modules/benchpress/src/sampler.js index 07be72e354..d43fd304ae 100644 --- a/modules/benchpress/src/sampler.js +++ b/modules/benchpress/src/sampler.js @@ -136,7 +136,10 @@ var _BINDINGS = [ execute: execute, time: time }), - [WebDriverAdapter, WebDriverExtension, Metric, Reporter, Validator, Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, _TIME] + [ + WebDriverAdapter, WebDriverExtension, Metric, Reporter, Validator, + Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, _TIME + ] ), bind(Options.FORCE_GC).toValue(false), bind(Options.PREPARE).toValue(false), diff --git a/modules/benchpress/test/metric/perflog_metric_spec.js b/modules/benchpress/test/metric/perflog_metric_spec.js index 61dffee193..8bf61ad3d4 100644 --- a/modules/benchpress/test/metric/perflog_metric_spec.js +++ b/modules/benchpress/test/metric/perflog_metric_spec.js @@ -2,8 +2,9 @@ import {ddescribe, describe, it, iit, xit, expect, beforeEach, afterEach} from ' import { List, ListWrapper } from 'angular2/src/facade/collection'; import { PromiseWrapper, Promise } from 'angular2/src/facade/async'; +import { isPresent } from 'angular2/src/facade/lang'; -import { Metric, PerflogMetric, WebDriverExtension, bind, Injector } from 'benchpress/common'; +import { Metric, PerflogMetric, WebDriverExtension, bind, Injector, Options } from 'benchpress/common'; import { TraceEventFactory } from '../trace_event_factory'; @@ -11,16 +12,18 @@ export function main() { var commandLog; var eventFactory = new TraceEventFactory('timeline', 'pid0'); - function createMetric(perfLogs) { + function createMetric(perfLogs, microIterations = 0) { commandLog = []; - return new Injector([ + var bindings = [ PerflogMetric.BINDINGS, bind(PerflogMetric.SET_TIMEOUT).toValue( (fn, millis) => { ListWrapper.push(commandLog, ['setTimeout', millis]); fn(); }), - bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog)) - ]).get(PerflogMetric); + bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog)), + bind(Options.MICRO_ITERATIONS).toValue(microIterations) + ]; + return new Injector(bindings).get(PerflogMetric); } describe('perflog metric', () => { @@ -151,10 +154,10 @@ export function main() { describe('aggregation', () => { - function aggregate(events) { + function aggregate(events, microIterations = 0) { ListWrapper.insert(events, 0, eventFactory.markStart('benchpress0', 0)); ListWrapper.push(events, eventFactory.markEnd('benchpress0', 10)); - var metric = createMetric([events]); + var metric = createMetric([events], microIterations); return metric .beginMeasure().then( (_) => metric.endMeasure(false) ); } @@ -252,6 +255,31 @@ export function main() { }); }); + describe('microIterations', () => { + + it('should not report scriptMicroAvg if microIterations = 0', (done) => { + aggregate([ + eventFactory.start('script', 0), + eventFactory.end('script', 5) + ], 0).then((data) => { + expect(isPresent(data['scriptMicroAvg'])).toBe(false); + done(); + }); + }); + + it('should report scriptMicroAvg', (done) => { + aggregate([ + eventFactory.start('script', 0), + eventFactory.end('script', 5) + ], 4).then((data) => { + expect(data['script']).toBe(5); + expect(data['scriptMicroAvg']).toBe(5/4); + done(); + }); + }); + + }); + describe('gcTimeInScript / gcAmountInScript', () => { it('should detect gc during script execution with begin/end events', (done) => { diff --git a/modules/benchpress/test/runner_spec.js b/modules/benchpress/test/runner_spec.js index 1834f27295..39dd5e5adf 100644 --- a/modules/benchpress/test/runner_spec.js +++ b/modules/benchpress/test/runner_spec.js @@ -85,6 +85,13 @@ export function main() { }); }); + it('should bind Options.MICRO_ITERATIONS', (done) => { + createRunner().sample({id: 'someId', microIterations: 23}).then( (_) => { + expect(injector.get(Options.MICRO_ITERATIONS)).toEqual(23); + done(); + }); + }); + it('should overwrite bindings per sample call', (done) => { createRunner([ bind(Options.DEFAULT_DESCRIPTION).toValue({'a': 1}),