From 1d4ffd986dec67517970b6f384bd428ee2b3f2c9 Mon Sep 17 00:00:00 2001 From: Tobias Bosch Date: Fri, 20 Feb 2015 13:32:54 -0800 Subject: [PATCH] feat(bench press): allow multiple reporters, metrics and driver extensions. --- modules/benchpress/benchpress.js | 2 + modules/benchpress/src/metric.js | 9 +++ modules/benchpress/src/metric/multi_metric.js | 69 ++++++++++++++++ .../benchpress/src/metric/perflog_metric.js | 2 +- modules/benchpress/src/reporter.js | 9 +++ .../src/reporter/console_reporter.js | 2 +- .../benchpress/src/reporter/multi_reporter.js | 42 ++++++++++ modules/benchpress/src/runner.js | 28 ++++++- modules/benchpress/src/sample_description.js | 9 ++- modules/benchpress/src/sample_options.js | 18 +++-- modules/benchpress/src/validator.js | 9 +++ .../validator/regression_slope_validator.js | 2 +- .../src/validator/size_validator.js | 2 +- modules/benchpress/src/web_driver_adapter.js | 14 +++- .../benchpress/src/web_driver_extension.js | 40 +++++++++- .../src/webdriver/chrome_driver_extension.js | 9 ++- .../src/webdriver/ios_driver_extension.js | 9 ++- .../webdriver/selenium_webdriver_adapter.es6 | 26 ++++-- .../test/metric/multi_metric_spec.js | 80 +++++++++++++++++++ .../test/metric/perflog_metric_spec.js | 2 +- .../test/reporter/console_reporter_spec.js | 2 +- .../test/reporter/multi_reporter_spec.js | 78 ++++++++++++++++++ modules/benchpress/test/runner_spec.js | 39 ++++++--- .../regression_slope_validator_spec.js | 2 +- .../test/validator/size_validator_spec.js | 2 +- .../test/web_driver_extension_spec.js | 51 ++++++++++++ .../webdriver/chrome_driver_extension_spec.js | 41 ++++++++-- .../webdriver/ios_driver_extension_spec.js | 12 ++- protractor-shared.js | 12 +-- 29 files changed, 559 insertions(+), 63 deletions(-) create mode 100644 modules/benchpress/src/metric/multi_metric.js create mode 100644 modules/benchpress/src/reporter/multi_reporter.js create mode 100644 modules/benchpress/test/metric/multi_metric_spec.js create mode 100644 modules/benchpress/test/reporter/multi_reporter_spec.js create mode 100644 modules/benchpress/test/web_driver_extension_spec.js diff --git a/modules/benchpress/benchpress.js b/modules/benchpress/benchpress.js index d60603da35..c967724f8c 100644 --- a/modules/benchpress/benchpress.js +++ b/modules/benchpress/benchpress.js @@ -14,5 +14,7 @@ export { IOsDriverExtension } from './src/webdriver/ios_driver_extension'; export { Runner } from './src/runner'; export { Options } from './src/sample_options'; export { MeasureValues } from './src/measure_values'; +export { MultiMetric } from './src/metric/multi_metric'; +export { MultiReporter } from './src/reporter/multi_reporter'; export { bind, Injector, OpaqueToken } from 'angular2/di'; diff --git a/modules/benchpress/src/metric.js b/modules/benchpress/src/metric.js index a9edcdb6ee..23e11d3854 100644 --- a/modules/benchpress/src/metric.js +++ b/modules/benchpress/src/metric.js @@ -1,3 +1,4 @@ +import { bind } from 'angular2/di'; import { Promise, PromiseWrapper } from 'angular2/src/facade/async'; @@ -11,6 +12,14 @@ import { StringMap } from 'angular2/src/facade/collection'; */ @ABSTRACT() export class Metric { + static bindTo(delegateToken) { + return [ + bind(Metric).toFactory( + (delegate) => delegate, [delegateToken] + ) + ]; + } + /** * Starts measuring */ diff --git a/modules/benchpress/src/metric/multi_metric.js b/modules/benchpress/src/metric/multi_metric.js new file mode 100644 index 0000000000..99cde83c2d --- /dev/null +++ b/modules/benchpress/src/metric/multi_metric.js @@ -0,0 +1,69 @@ +import { bind, Injector, OpaqueToken } from 'angular2/di'; +import { List, ListWrapper, StringMapWrapper, StringMap } from 'angular2/src/facade/collection'; +import { Promise, PromiseWrapper } from 'angular2/src/facade/async'; + +import { Metric } from '../metric'; + +export class MultiMetric extends Metric { + static createBindings(childTokens) { + return [ + bind(_CHILDREN).toAsyncFactory( + (injector) => PromiseWrapper.all(ListWrapper.map(childTokens, (token) => injector.asyncGet(token) )), + [Injector] + ), + bind(MultiMetric).toFactory( + (children) => new MultiMetric(children), + [_CHILDREN] + ) + ]; + } + + _metrics:List; + + constructor(metrics) { + super(); + this._metrics = metrics; + } + + /** + * Starts measuring + */ + beginMeasure():Promise { + return PromiseWrapper.all(ListWrapper.map( + this._metrics, (metric) => metric.beginMeasure() + )); + } + + /** + * Ends measuring and reports the data + * since the begin call. + * @param restart: Whether to restart right after this. + */ + endMeasure(restart:boolean):Promise { + return PromiseWrapper.all(ListWrapper.map( + this._metrics, (metric) => metric.endMeasure(restart) + )).then( (values) => { + return mergeStringMaps(values); + }); + } + + /** + * Describes the metrics provided by this metric implementation. + * (e.g. units, ...) + */ + describe():StringMap { + return mergeStringMaps(this._metrics.map( (metric) => metric.describe() )); + } +} + +function mergeStringMaps(maps) { + var result = {}; + ListWrapper.forEach(maps, (map) => { + StringMapWrapper.forEach(map, (value, prop) => { + result[prop] = value; + }); + }); + return result; +} + +var _CHILDREN = new OpaqueToken('MultiMetric.children'); diff --git a/modules/benchpress/src/metric/perflog_metric.js b/modules/benchpress/src/metric/perflog_metric.js index da18a71d56..c15eb33821 100644 --- a/modules/benchpress/src/metric/perflog_metric.js +++ b/modules/benchpress/src/metric/perflog_metric.js @@ -155,7 +155,7 @@ var _MAX_RETRY_COUNT = 20; var _MARK_NAME_PREFIX = 'benchpress'; var _SET_TIMEOUT = new OpaqueToken('PerflogMetric.setTimeout'); var _BINDINGS = [ - bind(Metric).toFactory( + bind(PerflogMetric).toFactory( (driverExtension, setTimeout) => new PerflogMetric(driverExtension, setTimeout), [WebDriverExtension, _SET_TIMEOUT] ), diff --git a/modules/benchpress/src/reporter.js b/modules/benchpress/src/reporter.js index abff062ea2..ddeae7b139 100644 --- a/modules/benchpress/src/reporter.js +++ b/modules/benchpress/src/reporter.js @@ -1,3 +1,4 @@ +import { bind } from 'angular2/di'; import { Promise, PromiseWrapper } from 'angular2/src/facade/async'; @@ -12,6 +13,14 @@ import { MeasureValues } from './measure_values'; */ @ABSTRACT() export class Reporter { + static bindTo(delegateToken) { + return [ + bind(Reporter).toFactory( + (delegate) => delegate, [delegateToken] + ) + ]; + } + reportMeasureValues(values:MeasureValues):Promise { throw new BaseException('NYI'); } diff --git a/modules/benchpress/src/reporter/console_reporter.js b/modules/benchpress/src/reporter/console_reporter.js index 94cd88180b..6050611d41 100644 --- a/modules/benchpress/src/reporter/console_reporter.js +++ b/modules/benchpress/src/reporter/console_reporter.js @@ -109,7 +109,7 @@ export class ConsoleReporter extends Reporter { var _PRINT = new OpaqueToken('ConsoleReporter.print'); var _COLUMN_WIDTH = new OpaqueToken('ConsoleReporter.columnWidht'); var _BINDINGS = [ - bind(Reporter).toFactory( + bind(ConsoleReporter).toFactory( (columnWidth, sampleDescription, print) => new ConsoleReporter(columnWidth, sampleDescription, print), [_COLUMN_WIDTH, SampleDescription, _PRINT] ), diff --git a/modules/benchpress/src/reporter/multi_reporter.js b/modules/benchpress/src/reporter/multi_reporter.js new file mode 100644 index 0000000000..239dbc631c --- /dev/null +++ b/modules/benchpress/src/reporter/multi_reporter.js @@ -0,0 +1,42 @@ +import { bind, Injector, OpaqueToken } from 'angular2/di'; +import { List, ListWrapper } from 'angular2/src/facade/collection'; +import { Promise, PromiseWrapper } from 'angular2/src/facade/async'; + +import { MeasureValues } from '../measure_values'; +import { Reporter } from '../reporter'; + +export class MultiReporter extends Reporter { + static createBindings(childTokens) { + return [ + bind(_CHILDREN).toAsyncFactory( + (injector) => PromiseWrapper.all(ListWrapper.map(childTokens, (token) => injector.asyncGet(token) )), + [Injector] + ), + bind(MultiReporter).toFactory( + (children) => new MultiReporter(children), + [_CHILDREN] + ) + ]; + } + + _reporters:List; + + constructor(reporters) { + super(); + this._reporters = reporters; + } + + reportMeasureValues(values:MeasureValues):Promise { + return PromiseWrapper.all(ListWrapper.map( + this._reporters, (reporter) => reporter.reportMeasureValues(values) + )); + } + + reportSample(completeSample:List, validSample:List):Promise { + return PromiseWrapper.all(ListWrapper.map( + this._reporters, (reporter) => reporter.reportSample(completeSample, validSample) + )); + } +} + +var _CHILDREN = new OpaqueToken('MultiReporter.children'); diff --git a/modules/benchpress/src/runner.js b/modules/benchpress/src/runner.js index d3f7983120..527be7dcc7 100644 --- a/modules/benchpress/src/runner.js +++ b/modules/benchpress/src/runner.js @@ -5,11 +5,19 @@ import { Promise } from 'angular2/src/facade/async'; import { Sampler, SampleState } from './sampler'; import { ConsoleReporter } from './reporter/console_reporter'; +import { MultiReporter } from './reporter/multi_reporter'; import { RegressionSlopeValidator } from './validator/regression_slope_validator'; +import { SizeValidator } from './validator/size_validator'; +import { Validator } from './validator'; import { PerflogMetric } from './metric/perflog_metric'; +import { MultiMetric } from './metric/multi_metric'; import { ChromeDriverExtension } from './webdriver/chrome_driver_extension'; +import { IOsDriverExtension } from './webdriver/ios_driver_extension'; +import { WebDriverExtension } from './web_driver_extension'; import { SampleDescription } from './sample_description'; - +import { WebDriverAdapter } from './web_driver_adapter'; +import { Reporter } from './reporter'; +import { Metric } from './metric'; import { Options } from './sample_options'; /** @@ -48,7 +56,23 @@ var _DEFAULT_BINDINGS = [ Sampler.BINDINGS, ConsoleReporter.BINDINGS, RegressionSlopeValidator.BINDINGS, + SizeValidator.BINDINGS, ChromeDriverExtension.BINDINGS, + IOsDriverExtension.BINDINGS, PerflogMetric.BINDINGS, - SampleDescription.BINDINGS + SampleDescription.BINDINGS, + MultiReporter.createBindings([ConsoleReporter]), + MultiMetric.createBindings([PerflogMetric]), + + Reporter.bindTo(MultiReporter), + Validator.bindTo(RegressionSlopeValidator), + WebDriverExtension.bindTo([ChromeDriverExtension, IOsDriverExtension]), + Metric.bindTo(MultiMetric), + + bind(Options.CAPABILITIES).toAsyncFactory( + (adapter) => adapter.capabilities(), [WebDriverAdapter] + ), + bind(Options.USER_AGENT).toAsyncFactory( + (adapter) => adapter.executeScript('return window.navigator.userAgent;'), [WebDriverAdapter] + ) ]; diff --git a/modules/benchpress/src/sample_description.js b/modules/benchpress/src/sample_description.js index 6a0538ccb3..7054741f4e 100644 --- a/modules/benchpress/src/sample_description.js +++ b/modules/benchpress/src/sample_description.js @@ -27,15 +27,18 @@ export class SampleDescription { var _BINDINGS = [ bind(SampleDescription).toFactory( - (metric, id, forceGc, validator, defaultDesc, userDesc) => new SampleDescription(id, + (metric, id, forceGc, userAgent, validator, defaultDesc, userDesc) => new SampleDescription(id, [ - {'forceGc': forceGc}, + {'forceGc': forceGc, 'userAgent': userAgent}, validator.describe(), defaultDesc, userDesc ], metric.describe()), - [Metric, Options.SAMPLE_ID, Options.FORCE_GC, Validator, Options.DEFAULT_DESCRIPTION, Options.SAMPLE_DESCRIPTION] + [ + Metric, Options.SAMPLE_ID, Options.FORCE_GC, Options.USER_AGENT, + Validator, Options.DEFAULT_DESCRIPTION, Options.SAMPLE_DESCRIPTION + ] ), bind(Options.DEFAULT_DESCRIPTION).toValue({}), bind(Options.SAMPLE_DESCRIPTION).toValue({}) diff --git a/modules/benchpress/src/sample_options.js b/modules/benchpress/src/sample_options.js index 7942ccfdd6..e403950137 100644 --- a/modules/benchpress/src/sample_options.js +++ b/modules/benchpress/src/sample_options.js @@ -13,11 +13,17 @@ export class Options { static get PREPARE() { return _PREPARE; } // TODO(tbosch): use static initializer when our transpiler supports it static get EXECUTE() { return _EXECUTE; } + // TODO(tbosch): use static initializer when our transpiler supports it + static get CAPABILITIES() { return _CAPABILITIES; } + // TODO(tbosch): use static initializer when our transpiler supports it + static get USER_AGENT() { return _USER_AGENT; } } -var _SAMPLE_ID = new OpaqueToken('SampleDescription.sampleId'); -var _DEFAULT_DESCRIPTION = new OpaqueToken('SampleDescription.defaultDescription'); -var _SAMPLE_DESCRIPTION = new OpaqueToken('SampleDescription.sampleDescription'); -var _FORCE_GC = new OpaqueToken('Sampler.forceGc'); -var _PREPARE = new OpaqueToken('Sampler.prepare'); -var _EXECUTE = new OpaqueToken('Sampler.execute'); +var _SAMPLE_ID = new OpaqueToken('Options.sampleId'); +var _DEFAULT_DESCRIPTION = new OpaqueToken('Options.defaultDescription'); +var _SAMPLE_DESCRIPTION = new OpaqueToken('Options.sampleDescription'); +var _FORCE_GC = new OpaqueToken('Options.forceGc'); +var _PREPARE = new OpaqueToken('Options.prepare'); +var _EXECUTE = new OpaqueToken('Options.execute'); +var _CAPABILITIES = new OpaqueToken('Options.capabilities'); +var _USER_AGENT = new OpaqueToken('Options.userAgent'); diff --git a/modules/benchpress/src/validator.js b/modules/benchpress/src/validator.js index 95311dcabd..04072018dc 100644 --- a/modules/benchpress/src/validator.js +++ b/modules/benchpress/src/validator.js @@ -1,3 +1,4 @@ +import { bind } from 'angular2/di'; import { List, StringMap } from 'angular2/src/facade/collection'; import { ABSTRACT, BaseException @@ -12,6 +13,14 @@ import { MeasureValues } from './measure_values'; */ @ABSTRACT() export class Validator { + static bindTo(delegateToken) { + return [ + bind(Validator).toFactory( + (delegate) => delegate, [delegateToken] + ) + ]; + } + /** * Calculates a valid sample out of the complete sample */ diff --git a/modules/benchpress/src/validator/regression_slope_validator.js b/modules/benchpress/src/validator/regression_slope_validator.js index 94a49ea286..3143a896f5 100644 --- a/modules/benchpress/src/validator/regression_slope_validator.js +++ b/modules/benchpress/src/validator/regression_slope_validator.js @@ -60,7 +60,7 @@ export class RegressionSlopeValidator extends Validator { var _SAMPLE_SIZE = new OpaqueToken('RegressionSlopeValidator.sampleSize'); var _METRIC = new OpaqueToken('RegressionSlopeValidator.metric'); var _BINDINGS = [ - bind(Validator).toFactory( + bind(RegressionSlopeValidator).toFactory( (sampleSize, metric) => new RegressionSlopeValidator(sampleSize, metric), [_SAMPLE_SIZE, _METRIC] ), diff --git a/modules/benchpress/src/validator/size_validator.js b/modules/benchpress/src/validator/size_validator.js index 652da00238..cbc71550b3 100644 --- a/modules/benchpress/src/validator/size_validator.js +++ b/modules/benchpress/src/validator/size_validator.js @@ -38,7 +38,7 @@ export class SizeValidator extends Validator { var _SAMPLE_SIZE = new OpaqueToken('SizeValidator.sampleSize'); var _BINDINGS = [ - bind(Validator).toFactory( + bind(SizeValidator).toFactory( (size) => new SizeValidator(size), [_SAMPLE_SIZE] ), diff --git a/modules/benchpress/src/web_driver_adapter.js b/modules/benchpress/src/web_driver_adapter.js index 1528dd2e13..9e27e06817 100644 --- a/modules/benchpress/src/web_driver_adapter.js +++ b/modules/benchpress/src/web_driver_adapter.js @@ -1,5 +1,7 @@ +import { bind } from 'angular2/di'; import { Promise } from 'angular2/src/facade/async'; import { BaseException, ABSTRACT } from 'angular2/src/facade/lang'; +import { List, Map } from 'angular2/src/facade/collection'; /** * A WebDriverAdapter bridges API differences between different WebDriver clients, @@ -8,16 +10,24 @@ import { BaseException, ABSTRACT } from 'angular2/src/facade/lang'; */ @ABSTRACT() export class WebDriverAdapter { + static bindTo(delegateToken) { + return [ + bind(WebDriverAdapter).toFactory( + (delegate) => delegate, [delegateToken] + ) + ]; + } + waitFor(callback:Function):Promise { throw new BaseException('NYI'); } executeScript(script:string):Promise { throw new BaseException('NYI'); } - capabilities():Promise { + capabilities():Promise { throw new BaseException('NYI'); } - logs(type:string):Promise { + logs(type:string):Promise { throw new BaseException('NYI'); } } diff --git a/modules/benchpress/src/web_driver_extension.js b/modules/benchpress/src/web_driver_extension.js index 79468c211b..b2849c6dfc 100644 --- a/modules/benchpress/src/web_driver_extension.js +++ b/modules/benchpress/src/web_driver_extension.js @@ -1,6 +1,10 @@ -import { BaseException, ABSTRACT } from 'angular2/src/facade/lang'; -import { Promise } from 'angular2/src/facade/async'; -import { List } from 'angular2/src/facade/collection'; +import { bind, Injector, OpaqueToken } from 'angular2/di'; + +import { BaseException, ABSTRACT, isBlank } from 'angular2/src/facade/lang'; +import { Promise, PromiseWrapper } from 'angular2/src/facade/async'; +import { List, ListWrapper, StringMap } from 'angular2/src/facade/collection'; + +import { Options } from './sample_options'; /** * A WebDriverExtension implements extended commands of the webdriver protocol @@ -9,6 +13,30 @@ import { List } from 'angular2/src/facade/collection'; */ @ABSTRACT() export class WebDriverExtension { + static bindTo(childTokens) { + return [ + bind(_CHILDREN).toAsyncFactory( + (injector) => PromiseWrapper.all(ListWrapper.map(childTokens, (token) => injector.asyncGet(token) )), + [Injector] + ), + bind(WebDriverExtension).toFactory( + (children, capabilities) => { + var delegate; + ListWrapper.forEach(children, (extension) => { + if (extension.supports(capabilities)) { + delegate = extension; + } + }); + if (isBlank(delegate)) { + throw new BaseException('Could not find a delegate for given capabilities!'); + } + return delegate; + }, + [_CHILDREN, Options.CAPABILITIES] + ) + ]; + } + gc():Promise { throw new BaseException('NYI'); } @@ -35,4 +63,10 @@ export class WebDriverExtension { readPerfLog():Promise { throw new BaseException('NYI'); } + + supports(capabilities:StringMap):boolean { + return true; + } } + +var _CHILDREN = new OpaqueToken('WebDriverExtension.children'); diff --git a/modules/benchpress/src/webdriver/chrome_driver_extension.js b/modules/benchpress/src/webdriver/chrome_driver_extension.js index 3a361fc478..ec9174d4fc 100644 --- a/modules/benchpress/src/webdriver/chrome_driver_extension.js +++ b/modules/benchpress/src/webdriver/chrome_driver_extension.js @@ -1,5 +1,5 @@ import { bind } from 'angular2/di'; -import { ListWrapper, StringMapWrapper } from 'angular2/src/facade/collection'; +import { ListWrapper, StringMapWrapper, StringMap } from 'angular2/src/facade/collection'; import { Json, isPresent, isBlank, RegExpWrapper, StringWrapper, BaseException, NumberWrapper } from 'angular2/src/facade/lang'; @@ -36,6 +36,7 @@ export class ChromeDriverExtension extends WebDriverExtension { return this._driver.executeScript(script); } + // See [Chrome Trace Event Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit) readPerfLog() { // TODO(tbosch): Bug in ChromeDriver: Need to execute at least one command // so that the browser logs can be read out! @@ -95,6 +96,10 @@ export class ChromeDriverExtension extends WebDriverExtension { }); return normalizedEvents; } + + supports(capabilities:StringMap):boolean { + return StringWrapper.equals(capabilities['browserName'].toLowerCase(), 'chrome'); + } } function normalizeEvent(chromeEvent, data) { @@ -120,7 +125,7 @@ function normalizeEvent(chromeEvent, data) { } var _BINDINGS = [ - bind(WebDriverExtension).toFactory( + bind(ChromeDriverExtension).toFactory( (driver) => new ChromeDriverExtension(driver), [WebDriverAdapter] ) diff --git a/modules/benchpress/src/webdriver/ios_driver_extension.js b/modules/benchpress/src/webdriver/ios_driver_extension.js index 48220c58a2..fdd2f5fac2 100644 --- a/modules/benchpress/src/webdriver/ios_driver_extension.js +++ b/modules/benchpress/src/webdriver/ios_driver_extension.js @@ -1,5 +1,5 @@ import { bind } from 'angular2/di'; -import { ListWrapper } from 'angular2/src/facade/collection'; +import { ListWrapper, StringMap } from 'angular2/src/facade/collection'; import { Json, isPresent, isBlank, RegExpWrapper, StringWrapper } from 'angular2/src/facade/lang'; @@ -36,6 +36,7 @@ export class IOsDriverExtension extends WebDriverExtension { return this._driver.executeScript(script); } + // See https://github.com/WebKit/webkit/tree/master/Source/WebInspectorUI/Versions readPerfLog() { // TODO(tbosch): Bug in IOsDriver: Need to execute at least one command // so that the browser logs can be read out! @@ -97,6 +98,10 @@ export class IOsDriverExtension extends WebDriverExtension { }); return events; } + + supports(capabilities:StringMap):boolean { + return StringWrapper.equals(capabilities['browserName'].toLowerCase(), 'safari'); + } } function createEvent(ph, name, time, args = null) { @@ -132,7 +137,7 @@ function createMarkEndEvent(name, time) { } var _BINDINGS = [ - bind(WebDriverExtension).toFactory( + bind(IOsDriverExtension).toFactory( (driver) => new IOsDriverExtension(driver), [WebDriverAdapter] ) diff --git a/modules/benchpress/src/webdriver/selenium_webdriver_adapter.es6 b/modules/benchpress/src/webdriver/selenium_webdriver_adapter.es6 index 89117cffc5..31bbd1cc29 100644 --- a/modules/benchpress/src/webdriver/selenium_webdriver_adapter.es6 +++ b/modules/benchpress/src/webdriver/selenium_webdriver_adapter.es6 @@ -17,7 +17,13 @@ export class SeleniumWebDriverAdapter extends WebDriverAdapter { _convertPromise(thenable) { var completer = PromiseWrapper.completer(); - thenable.then(completer.complete, completer.reject); + thenable.then( + // selenium-webdriver uses an own Node.js context, + // so we need to convert data into objects of this context. + // (e.g. otherwise instanceof checks of rtts_assert would fail) + (data) => completer.complete(convertToLocalProcess(data)), + completer.reject + ); return completer.promise; } @@ -30,7 +36,9 @@ export class SeleniumWebDriverAdapter extends WebDriverAdapter { } capabilities():Promise { - return this._convertPromise(this._driver.getCapabilities()); + return this._convertPromise( + this._driver.getCapabilities().then( (capsObject) => capsObject.toJSON() ) + ); } logs(type:string):Promise { @@ -39,11 +47,15 @@ export class SeleniumWebDriverAdapter extends WebDriverAdapter { return this._convertPromise(this._driver.schedule( new webdriver.Command(webdriver.CommandName.GET_LOG). setParameter('type', type), - 'WebDriver.manage().logs().get(' + type + ')').then( (logs) => { - // Need to convert the Array into an instance of an Array - // as selenium-webdriver uses an own Node.js context! - return [].slice.call(logs); - })); + 'WebDriver.manage().logs().get(' + type + ')')); } } + +function convertToLocalProcess(data) { + var serialized = JSON.stringify(data); + if (''+serialized === 'undefined') { + return undefined; + } + return JSON.parse(serialized); +} \ No newline at end of file diff --git a/modules/benchpress/test/metric/multi_metric_spec.js b/modules/benchpress/test/metric/multi_metric_spec.js new file mode 100644 index 0000000000..8a7b5841f0 --- /dev/null +++ b/modules/benchpress/test/metric/multi_metric_spec.js @@ -0,0 +1,80 @@ +import {ddescribe, describe, it, iit, xit, expect, beforeEach, afterEach} from 'angular2/test_lib'; + +import { List, ListWrapper, StringMap } from 'angular2/src/facade/collection'; +import { PromiseWrapper, Promise } from 'angular2/src/facade/async'; + +import { Metric, MultiMetric, bind, Injector } from 'benchpress/benchpress'; + +export function main() { + function createMetric(ids) { + return new Injector([ + ListWrapper.map(ids, (id) => bind(id).toValue(new MockMetric(id)) ), + MultiMetric.createBindings(ids) + ]).asyncGet(MultiMetric); + } + + describe('multi metric', () => { + + it('should merge descriptions', (done) => { + createMetric(['m1', 'm2']).then( (m) => { + expect(m.describe()).toEqual({ + 'm1': 'describe', 'm2': 'describe' + }); + done(); + }); + }); + + it('should merge all beginMeasure calls', (done) => { + createMetric(['m1', 'm2']) + .then( (m) => m.beginMeasure() ) + .then( (values) => { + expect(values).toEqual([ + 'm1_beginMeasure', 'm2_beginMeasure' + ]); + done(); + }); + }); + + [false, true].forEach( (restartFlag) => { + it(`should merge all endMeasure calls for restart=${restartFlag}`, (done) => { + createMetric(['m1', 'm2']) + .then( (m) => m.endMeasure(restartFlag) ) + .then( (values) => { + expect(values).toEqual({ + 'm1': { 'restart': restartFlag }, + 'm2': { 'restart': restartFlag } + }); + done(); + }); + }); + }); + + }); +} + +class MockMetric extends Metric { + _id:string; + + constructor(id) { + super(); + this._id = id; + } + + beginMeasure():Promise { + return PromiseWrapper.resolve(`${this._id}_beginMeasure`); + } + + endMeasure(restart:boolean):Promise { + var result = {}; + result[this._id] = { + 'restart': restart + }; + return PromiseWrapper.resolve(result); + } + + describe():StringMap { + var result = {}; + result[this._id] = 'describe'; + return result; + } +} diff --git a/modules/benchpress/test/metric/perflog_metric_spec.js b/modules/benchpress/test/metric/perflog_metric_spec.js index 37d1696b29..bccfcbfa41 100644 --- a/modules/benchpress/test/metric/perflog_metric_spec.js +++ b/modules/benchpress/test/metric/perflog_metric_spec.js @@ -20,7 +20,7 @@ export function main() { fn(); }), bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog)) - ]).get(Metric); + ]).get(PerflogMetric); } describe('perflog metric', () => { diff --git a/modules/benchpress/test/reporter/console_reporter_spec.js b/modules/benchpress/test/reporter/console_reporter_spec.js index aff49414e4..00c544bc43 100644 --- a/modules/benchpress/test/reporter/console_reporter_spec.js +++ b/modules/benchpress/test/reporter/console_reporter_spec.js @@ -29,7 +29,7 @@ export function main() { if (isPresent(columnWidth)) { ListWrapper.push(bindings, bind(ConsoleReporter.COLUMN_WIDTH).toValue(columnWidth)); } - reporter = new Injector(bindings).get(Reporter); + reporter = new Injector(bindings).get(ConsoleReporter); } it('should print the sample id, description and table header', () => { diff --git a/modules/benchpress/test/reporter/multi_reporter_spec.js b/modules/benchpress/test/reporter/multi_reporter_spec.js new file mode 100644 index 0000000000..84906764df --- /dev/null +++ b/modules/benchpress/test/reporter/multi_reporter_spec.js @@ -0,0 +1,78 @@ +import {ddescribe, describe, it, iit, xit, expect, beforeEach, afterEach} from 'angular2/test_lib'; + +import { List, ListWrapper, StringMap } from 'angular2/src/facade/collection'; +import { PromiseWrapper, Promise } from 'angular2/src/facade/async'; +import { DateWrapper } from 'angular2/src/facade/lang'; + +import { Reporter, MultiReporter, bind, Injector, MeasureValues } from 'benchpress/benchpress'; + +export function main() { + function createReporters(ids) { + return new Injector([ + ListWrapper.map(ids, (id) => bind(id).toValue(new MockReporter(id)) ), + MultiReporter.createBindings(ids) + ]).asyncGet(MultiReporter); + } + + describe('multi reporter', () => { + + it('should reportMeasureValues to all', (done) => { + var mv = new MeasureValues(0, DateWrapper.now(), {}); + createReporters(['m1', 'm2']) + .then( (r) => r.reportMeasureValues(mv) ) + .then( (values) => { + + expect(values).toEqual([ + {'id': 'm1', 'values': mv}, + {'id': 'm2', 'values': mv} + ]); + done(); + }); + }); + + it('should reportSample to call', (done) => { + var completeSample = [ + new MeasureValues(0, DateWrapper.now(), {}), + new MeasureValues(1, DateWrapper.now(), {}) + ]; + var validSample = [completeSample[1]]; + + createReporters(['m1', 'm2']) + .then( (r) => r.reportSample(completeSample, validSample) ) + .then( (values) => { + + expect(values).toEqual([ + {'id': 'm1', 'completeSample': completeSample, 'validSample': validSample}, + {'id': 'm2', 'completeSample': completeSample, 'validSample': validSample} + ]); + done(); + }) + }); + + }); +} + +class MockReporter extends Reporter { + _id:string; + + constructor(id) { + super(); + this._id = id; + } + + reportMeasureValues(values:MeasureValues):Promise { + return PromiseWrapper.resolve({ + 'id': this._id, + 'values': values + }); + } + + reportSample(completeSample:List, validSample:List):Promise { + return PromiseWrapper.resolve({ + 'id': this._id, + 'completeSample': completeSample, + 'validSample': validSample + }); + } + +} diff --git a/modules/benchpress/test/runner_spec.js b/modules/benchpress/test/runner_spec.js index 1b6a561561..193519af60 100644 --- a/modules/benchpress/test/runner_spec.js +++ b/modules/benchpress/test/runner_spec.js @@ -2,7 +2,7 @@ import {describe, it, iit, xit, expect, beforeEach, afterEach} from 'angular2/te import { Runner, Sampler, SampleDescription, Validator, bind, Injector, Metric, - Options + Options, WebDriverAdapter } from 'benchpress/benchpress'; import { isBlank } from 'angular2/src/facade/lang'; import { Promise, PromiseWrapper } from 'angular2/src/facade/async'; @@ -25,16 +25,19 @@ export function main() { }, [Injector] ), bind(Metric).toFactory( () => new MockMetric(), []), - bind(Validator).toFactory( () => new MockValidator(), []) + bind(Validator).toFactory( () => new MockValidator(), []), + bind(WebDriverAdapter).toFactory( () => new MockWebDriverAdapter(), []) ]); return runner; } it('should set SampleDescription.id', (done) => { - createRunner().sample({id: 'someId'}).then( (_) => { - expect(injector.get(SampleDescription).id).toBe('someId'); - done(); - }); + createRunner().sample({id: 'someId'}) + .then( (_) => injector.asyncGet(SampleDescription) ) + .then( (desc) => { + expect(desc.id).toBe('someId'); + done(); + }); }); it('should merge SampleDescription.description', (done) => { @@ -42,9 +45,12 @@ export function main() { bind(Options.DEFAULT_DESCRIPTION).toValue({'a': 1}) ]).sample({id: 'someId', bindings: [ bind(Options.SAMPLE_DESCRIPTION).toValue({'b': 2}) - ]}).then( (_) => { - expect(injector.get(SampleDescription).description).toEqual({ + ]}).then( (_) => injector.asyncGet(SampleDescription) ) + .then( (desc) => { + + expect(desc.description).toEqual({ 'forceGc': false, + 'userAgent': 'someUserAgent', 'a': 1, 'b': 2, 'v': 11 @@ -54,8 +60,11 @@ export function main() { }); it('should fill SampleDescription.metrics from the Metric', (done) => { - createRunner().sample({id: 'someId'}).then( (_) => { - expect(injector.get(SampleDescription).metrics).toEqual({ 'm1': 'some metric' }); + createRunner().sample({id: 'someId'}) + .then( (_) => injector.asyncGet(SampleDescription) ) + .then( (desc) => { + + expect(desc.metrics).toEqual({ 'm1': 'some metric' }); done(); }); }); @@ -81,7 +90,9 @@ export function main() { bind(Options.DEFAULT_DESCRIPTION).toValue({'a': 1}), ]).sample({id: 'someId', bindings: [ bind(Options.DEFAULT_DESCRIPTION).toValue({'a': 2}), - ]}).then( (_) => { + ]}).then( (_) => injector.asyncGet(SampleDescription) ) + .then( (desc) => { + expect(injector.get(SampleDescription).description['a']).toBe(2); done(); }); @@ -91,6 +102,12 @@ export function main() { }); } +class MockWebDriverAdapter extends WebDriverAdapter { + executeScript(script):Promise { + return PromiseWrapper.resolve('someUserAgent'); + } +} + class MockValidator extends Validator { constructor() { super(); diff --git a/modules/benchpress/test/validator/regression_slope_validator_spec.js b/modules/benchpress/test/validator/regression_slope_validator_spec.js index 5a78f72ab3..bf8570eef7 100644 --- a/modules/benchpress/test/validator/regression_slope_validator_spec.js +++ b/modules/benchpress/test/validator/regression_slope_validator_spec.js @@ -15,7 +15,7 @@ export function main() { RegressionSlopeValidator.BINDINGS, bind(RegressionSlopeValidator.METRIC).toValue(metric), bind(RegressionSlopeValidator.SAMPLE_SIZE).toValue(size) - ]).get(Validator); + ]).get(RegressionSlopeValidator); } it('should return sampleSize and metric as description', () => { diff --git a/modules/benchpress/test/validator/size_validator_spec.js b/modules/benchpress/test/validator/size_validator_spec.js index 147addc908..3e22222736 100644 --- a/modules/benchpress/test/validator/size_validator_spec.js +++ b/modules/benchpress/test/validator/size_validator_spec.js @@ -14,7 +14,7 @@ export function main() { validator = new Injector([ SizeValidator.BINDINGS, bind(SizeValidator.SAMPLE_SIZE).toValue(size) - ]).get(Validator); + ]).get(SizeValidator); } it('should return sampleSize as description', () => { diff --git a/modules/benchpress/test/web_driver_extension_spec.js b/modules/benchpress/test/web_driver_extension_spec.js new file mode 100644 index 0000000000..ae000c6fc0 --- /dev/null +++ b/modules/benchpress/test/web_driver_extension_spec.js @@ -0,0 +1,51 @@ +import {ddescribe, describe, it, iit, xit, expect, beforeEach, afterEach} from 'angular2/test_lib'; + +import { StringMap, ListWrapper } from 'angular2/src/facade/collection'; +import { isPresent, StringWrapper, isJsObject } from 'angular2/src/facade/lang'; + +import { WebDriverExtension, bind, Injector, Options } from 'benchpress/benchpress'; + +export function main() { + function createExtension(ids, caps) { + return new Injector([ + ListWrapper.map(ids, (id) => bind(id).toValue(new MockExtension(id)) ), + bind(Options.CAPABILITIES).toValue(caps), + WebDriverExtension.bindTo(ids) + ]).asyncGet(WebDriverExtension); + } + + describe('WebDriverExtension.bindTo', () => { + + it('should bind the extension that matches the capabilities', (done) => { + createExtension(['m1', 'm2', 'm3'], {'browser': 'm2'}).then( (m) => { + expect(m.id).toEqual('m2'); + done(); + }); + }); + + // TODO(tbosch): In Dart, somehow we don't provide the error + // correctly in the promise result... + if (isJsObject({})) { + it('should throw if there is no match', (done) => { + createExtension(['m1'], {'browser': 'm2'}).then(null, (err) => { + expect(isPresent(err)).toBe(true); + done(); + }); + }); + } + + }); +} + +class MockExtension extends WebDriverExtension { + id:string; + + constructor(id) { + super(); + this.id = id; + } + + supports(capabilities:StringMap):boolean { + return StringWrapper.equals(capabilities['browser'], this.id); + } +} diff --git a/modules/benchpress/test/webdriver/chrome_driver_extension_spec.js b/modules/benchpress/test/webdriver/chrome_driver_extension_spec.js index 061e696c99..adcece7389 100644 --- a/modules/benchpress/test/webdriver/chrome_driver_extension_spec.js +++ b/modules/benchpress/test/webdriver/chrome_driver_extension_spec.js @@ -2,7 +2,7 @@ import {describe, it, iit, xit, expect, beforeEach, afterEach} from 'angular2/te import { ListWrapper } from 'angular2/src/facade/collection'; import { PromiseWrapper } from 'angular2/src/facade/async'; -import { Json, isBlank } from 'angular2/src/facade/lang'; +import { Json, isBlank, isJsObject } from 'angular2/src/facade/lang'; import { WebDriverExtension, ChromeDriverExtension, @@ -20,15 +20,15 @@ export function main() { var chromeTimelineEvents = new TraceEventFactory('disabled-by-default-devtools.timeline', 'pid0'); var normEvents = new TraceEventFactory('timeline', 'pid0'); - function createExtension(perfRecords = null) { + function createExtension(perfRecords = null, messageMethod = 'Tracing.dataCollected') { if (isBlank(perfRecords)) { perfRecords = []; } log = []; extension = new Injector([ ChromeDriverExtension.BINDINGS, - bind(WebDriverAdapter).toValue(new MockDriverAdapter(log, perfRecords)) - ]).get(WebDriverExtension); + bind(WebDriverAdapter).toValue(new MockDriverAdapter(log, perfRecords, messageMethod)) + ]).get(ChromeDriverExtension); return extension; } @@ -153,6 +153,31 @@ export function main() { }); }); + // TODO(tbosch): In Dart, somehow we don't provide the error + // correctly in the promise result... + if (isJsObject({})) { + it('should throw an error on buffer overflow', (done) => { + createExtension([ + chromeTimelineEvents.start('FunctionCall', 1234), + ], 'Tracing.bufferUsage').readPerfLog().then(null, (err) => { + expect( () => { + throw err; + }).toThrowError('The DevTools trace buffer filled during the test!'); + done(); + }); + }); + } + + it('should match chrome browsers', () => { + expect(createExtension().supports({ + 'browserName': 'chrome' + })).toBe(true); + + expect(createExtension().supports({ + 'browserName': 'Chrome' + })).toBe(true); + }); + }); }); @@ -161,10 +186,12 @@ export function main() { class MockDriverAdapter extends WebDriverAdapter { _log:List; _events:List; - constructor(log, events) { + _messageMethod:string; + constructor(log, events, messageMethod) { super(); this._log = log; this._events = events; + this._messageMethod = messageMethod; } executeScript(script) { @@ -175,11 +202,11 @@ class MockDriverAdapter extends WebDriverAdapter { logs(type) { ListWrapper.push(this._log, ['logs', type]); if (type === 'performance') { - return PromiseWrapper.resolve(this._events.map(function(event) { + return PromiseWrapper.resolve(this._events.map( (event) => { return { 'message': Json.stringify({ 'message': { - 'method': 'Tracing.dataCollected', + 'method': this._messageMethod, 'params': event } }) diff --git a/modules/benchpress/test/webdriver/ios_driver_extension_spec.js b/modules/benchpress/test/webdriver/ios_driver_extension_spec.js index a6bd8b52f0..7a1d8629b9 100644 --- a/modules/benchpress/test/webdriver/ios_driver_extension_spec.js +++ b/modules/benchpress/test/webdriver/ios_driver_extension_spec.js @@ -26,7 +26,7 @@ export function main() { extension = new Injector([ IOsDriverExtension.BINDINGS, bind(WebDriverAdapter).toValue(new MockDriverAdapter(log, perfRecords)) - ]).get(WebDriverExtension); + ]).get(IOsDriverExtension); return extension; } @@ -155,6 +155,16 @@ export function main() { }); }); + it('should match safari browsers', () => { + expect(createExtension().supports({ + 'browserName': 'safari' + })).toBe(true); + + expect(createExtension().supports({ + 'browserName': 'Safari' + })).toBe(true); + }); + }); }); diff --git a/protractor-shared.js b/protractor-shared.js index 73856b8b6d..55e5f67679 100644 --- a/protractor-shared.js +++ b/protractor-shared.js @@ -175,19 +175,13 @@ exports.createBenchpressRunner = function(options) { benchpress.bind(benchpress.Options.DEFAULT_DESCRIPTION).toValue({ 'lang': options.lang, 'runId': runId - }), - // TODO(tbosch): Make the ChromeDriverExtension configurable based on the - // capabilities. Should support the case where we test against - // ios and chrome at the same time! - benchpress.bind(benchpress.WebDriverExtension).toFactory(function(adapter) { - return new benchpress.ChromeDriverExtension(adapter); - }, [benchpress.WebDriverAdapter]) + }) ]; if (argv['benchmark']) { - bindings.push(benchpress.RegressionSlopeValidator.BINDINGS); + bindings.push(benchpress.Validator.bindTo(benchpress.RegressionSlopeValidator)); bindings.push(benchpress.bind(benchpress.RegressionSlopeValidator.SAMPLE_SIZE).toValue(argv['sample-size'])); } else { - bindings.push(benchpress.SizeValidator.BINDINGS); + bindings.push(benchpress.Validator.bindTo(benchpress.SizeValidator)); bindings.push(benchpress.bind(benchpress.SizeValidator.SAMPLE_SIZE).toValue(1)); }