From 6dceaf209d7dccc320361ef874ac2f0fe062704a Mon Sep 17 00:00:00 2001 From: Tobias Bosch Date: Tue, 30 Aug 2016 09:29:39 -0700 Subject: [PATCH] fix(benchmarks): recreate setup for running benchmarks --- build.sh | 8 +- gulpfile.js | 2 +- modules/@angular/benchpress/tsconfig-es5.json | 7 +- modules/benchmarks/README.md | 16 ++- modules/benchmarks/e2e_test/tree_perf.ts | 53 +++++--- modules/benchmarks/e2e_test/tree_spec.ts | 43 +++++++ modules/e2e_util/e2e_util.ts | 32 ++++- modules/e2e_util/perf_util.ts | 117 +++++++++--------- protractor-e2e.conf.js | 6 +- protractor-perf.conf.js | 57 +++++++++ scripts/ci-lite/build.sh | 1 - scripts/ci-lite/test_e2e.sh | 3 +- tools/tsc-watch/index.ts | 3 +- 13 files changed, 254 insertions(+), 94 deletions(-) create mode 100644 modules/benchmarks/e2e_test/tree_spec.ts create mode 100644 protractor-perf.conf.js diff --git a/build.sh b/build.sh index a6c78bb385..546e3c189a 100755 --- a/build.sh +++ b/build.sh @@ -69,7 +69,8 @@ for PACKAGE in \ http \ router \ upgrade \ - compiler-cli + compiler-cli \ + benchpress do PWD=`pwd` SRCDIR=${PWD}/modules/@angular/${PACKAGE} @@ -101,7 +102,7 @@ do find ${DESTDIR} -type f -name '*.d.ts' -print0 | xargs -0 sed -i -E 's/^( +)abstract ([[:alnum:]]+\:)/\1\2/g' fi - if [[ ${PACKAGE} != compiler-cli ]]; then + if [[ ${PACKAGE} != compiler-cli && ${PACKAGE} != benchpress ]]; then echo "====== BUNDLING: ${SRCDIR} =====" mkdir ${DESTDIR}/bundles @@ -128,6 +129,3 @@ do fi done - -echo "====== COMPILING: \$(npm bin)/tsc -p benchpress/tsconfig.json =====" -$(npm bin)/tsc -p ./modules/benchpress/tsconfig.json diff --git a/gulpfile.js b/gulpfile.js index 477446f430..9f4eb31fb3 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -12,7 +12,7 @@ const os = require('os'); const srcsToFmt = ['tools/**/*.ts', 'modules/@angular/**/*.ts', '!tools/public_api_guard/**/*.d.ts', - 'modules/benchpress/**/*.ts', 'modules/playground/**/*.ts', 'modules/benchmarks/**/*.ts']; + 'modules/playground/**/*.ts', 'modules/benchmarks/**/*.ts', 'modules/e2e_util/**/*.ts']; gulp.task('format:enforce', () => { const format = require('gulp-clang-format'); diff --git a/modules/@angular/benchpress/tsconfig-es5.json b/modules/@angular/benchpress/tsconfig-es5.json index cea15f38e2..f21137b9f1 100644 --- a/modules/@angular/benchpress/tsconfig-es5.json +++ b/modules/@angular/benchpress/tsconfig-es5.json @@ -7,7 +7,8 @@ "sourceMap": true, "baseUrl": ".", "paths": { - "@angular/core": ["../../../dist/packages-dist/core"] + "@angular/core": ["../../../dist/packages-dist/core"], + "selenium-webdriver": ["../../../node_modules/@types/selenium-webdriver/index.d.ts"] }, "experimentalDecorators": true, "rootDir": ".", @@ -20,7 +21,7 @@ "index.ts", "../../../node_modules/@types/node/index.d.ts", "../../../node_modules/@types/jasmine/index.d.ts", - "../../node_modules/@types/protractor/index.d.ts", - "../../node_modules/@types/selenium-webdriver/index.d.ts" + "../../../node_modules/@types/protractor/index.d.ts", + "../../../node_modules/zone.js/dist/zone.js.d.ts" ] } diff --git a/modules/benchmarks/README.md b/modules/benchmarks/README.md index ff39f5434a..acc3ec72a4 100644 --- a/modules/benchmarks/README.md +++ b/modules/benchmarks/README.md @@ -1,7 +1,7 @@ # How to run the benchmarks locally ## Run in the browser -$ build.sh +$ build.sh (only needed 1x to copy over third party resources) $ cp -r ./modules/benchmarks ./dist/all/ $ ./node_modules/.bin/tsc -p modules --emitDecoratorMetadata -w $ gulp serve @@ -9,4 +9,16 @@ $ open http://localhost:8000/all/benchmarks/src/tree/ng2/index.html?bundles=fals ## Run e2e tests $ export NODE_PATH=$(pwd)/dist/all:$(pwd)/dist/tools -$ ./node_modules/.bin/protractor protractor-js-new-world.conf.js --specs=dist/all/benchmarks/e2e_test/tree_perf.js +$ ./node_modules/.bin/protractor protractor-e2e.conf.js --specs=dist/all/benchmarks/e2e_test/tree_spec.js + +Options for protractor with `protractor-e2e.conf.js`: +- `--bundles=true`: use prebuilt bundles +- `--ng-help`: show all available options + +## Run benchmarks tests +$ export NODE_PATH=$(pwd)/dist/all:$(pwd)/dist/tools +$ ./node_modules/.bin/protractor protractor-perf.conf.js --specs=dist/all/benchmarks/e2e_test/tree_perf.js + +Options for protractor with `protractor-perf.conf.js`: +- `--bundles=true`: use prebuilt bundles +- `--ng-help`: show all available options diff --git a/modules/benchmarks/e2e_test/tree_perf.ts b/modules/benchmarks/e2e_test/tree_perf.ts index 5a7c770987..8fd12960a5 100644 --- a/modules/benchmarks/e2e_test/tree_perf.ts +++ b/modules/benchmarks/e2e_test/tree_perf.ts @@ -6,32 +6,47 @@ * found in the LICENSE file at https://angular.io/license */ -import {verifyNoBrowserErrors} from 'e2e_util/e2e_util'; +import {runBenchmark, verifyNoBrowserErrors} from 'e2e_util/perf_util'; -const useBundles = false; - -describe('tree benchmark', function() { +describe('tree benchmark', () => { afterEach(verifyNoBrowserErrors); - it('should work for the baseline', function() { - browser.ignoreSynchronization = true; - browser.get(`all/benchmarks/src/tree/baseline/index.html?bundles=${useBundles}`); - $('#createDom').click(); - expect($('baseline').getText()).toContain('0'); + it('should work for the baseline', function(done) { + runBenchmark({ + id: 'deepTree.baseline', + url: 'all/benchmarks/src/tree/baseline/index.html', + ignoreBrowserSynchronization: true, + params: [{name: 'depth', value: 9}], + work: () => { + $('#createDom').click(); + $('#destroyDom').click(); + } + }).then(done, done.fail); }); - it('should work for ng2', function() { - browser.get(`all/benchmarks/src/tree/ng2/index.html?bundles=${useBundles}`); - $('#createDom').click(); - expect($('app').getText()).toContain('0'); + it('should work for ng2', function(done) { + runBenchmark({ + id: 'deepTree.ng2', + url: 'all/benchmarks/src/tree/ng2/index.html', + params: [{name: 'depth', value: 9}], + work: () => { + $('#createDom').click(); + $('#destroyDom').click(); + } + }).then(done, done.fail) }); - it('should work for polymer', function() { - browser.ignoreSynchronization = true; - browser.get(`all/benchmarks/src/tree/polymer/index.html?bundles=${useBundles}`); - $('#createDom').click(); - expect($('#app').getText()).toContain('0'); + it('should work for polymer', function(done) { + runBenchmark({ + id: 'deepTree.polymer', + url: 'all/benchmarks/src/tree/polymer/index.html', + ignoreBrowserSynchronization: true, + params: [{name: 'depth', value: 9}], + work: () => { + $('#createDom').click(); + $('#destroyDom').click(); + } + }).then(done, done.fail) }); - }); diff --git a/modules/benchmarks/e2e_test/tree_spec.ts b/modules/benchmarks/e2e_test/tree_spec.ts new file mode 100644 index 0000000000..3b8acf2fbb --- /dev/null +++ b/modules/benchmarks/e2e_test/tree_spec.ts @@ -0,0 +1,43 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {openBrowser, verifyNoBrowserErrors} from 'e2e_util/e2e_util'; + +const useBundles = false; + +describe('tree benchmark', function() { + + afterEach(verifyNoBrowserErrors); + + it('should work for the baseline', function() { + openBrowser({ + url: 'all/benchmarks/src/tree/baseline/index.html', + ignoreBrowserSynchronization: true, + }); + $('#createDom').click(); + expect($('baseline').getText()).toContain('0'); + }); + + it('should work for ng2', function() { + openBrowser({ + url: 'all/benchmarks/src/tree/ng2/index.html', + }); + $('#createDom').click(); + expect($('app').getText()).toContain('0'); + }); + + it('should work for polymer', function() { + openBrowser({ + url: 'all/benchmarks/src/tree/polymer/index.html', + ignoreBrowserSynchronization: true, + }); + $('#createDom').click(); + expect($('#app').getText()).toContain('0'); + }); + +}); diff --git a/modules/e2e_util/e2e_util.ts b/modules/e2e_util/e2e_util.ts index bf4eb16cfc..9ac0250940 100644 --- a/modules/e2e_util/e2e_util.ts +++ b/modules/e2e_util/e2e_util.ts @@ -5,13 +5,43 @@ * Use of this source code is governed by an MIT-style license that can be * found in the LICENSE file at https://angular.io/license */ - +const yargs = require('yargs'); import * as webdriver from 'selenium-webdriver'; +let cmdArgs: {'bundles': boolean}; declare var browser: any; declare var expect: any; +export function readCommandLine(extraOptions?: {[key: string]: any}) { + const options: {[key: string]: any} = { + 'bundles': {describe: 'Whether to use the angular bundles or not.', default: false} + }; + for (var key in extraOptions) { + options[key] = extraOptions[key]; + } + + cmdArgs = yargs.usage('Angular e2e test options.').options(options).help('ng-help').wrap(40).argv; + return cmdArgs; +} + +export function openBrowser(config: { + url: string, + params?: {name: string, value: any}[], + ignoreBrowserSynchronization?: boolean +}) { + if (config.ignoreBrowserSynchronization) { + browser.ignoreSynchronization = true; + } + var params = config.params || []; + params = params.concat([{name: 'bundles', value: cmdArgs.bundles}]); + + var urlParams: string[] = []; + params.forEach((param) => { urlParams.push(param.name + '=' + param.value); }); + var url = encodeURI(config.url + '?' + urlParams.join('&')); + browser.get(url); +} + /** * @experimental This API will be moved to Protractor. */ diff --git a/modules/e2e_util/perf_util.ts b/modules/e2e_util/perf_util.ts index 2ebcdf778e..1214a76861 100644 --- a/modules/e2e_util/perf_util.ts +++ b/modules/e2e_util/perf_util.ts @@ -5,70 +5,73 @@ * Use of this source code is governed by an MIT-style license that can be * found in the LICENSE file at https://angular.io/license */ - export {verifyNoBrowserErrors} from './e2e_util'; -var benchpress = (global as any /** TODO #9100 */)['benchpress']; -var bind = benchpress.bind; -var Options = benchpress.Options; +const yargs = require('yargs'); +const nodeUuid = require('node-uuid'); +import * as fs from 'fs-extra'; -export function runClickBenchmark(config: any /** TODO #9100 */) { - browser.ignoreSynchronization = !config.waitForAngular2; - var buttons = - config.buttons.map(function(selector: any /** TODO #9100 */) { return $(selector); }); - config.work = function() { - buttons.forEach(function(button: any /** TODO #9100 */) { button.click(); }); - }; - return runBenchmark(config); +import {SeleniumWebDriverAdapter, Options, JsonFileReporter, Validator, RegressionSlopeValidator, ConsoleReporter, SizeValidator, MultiReporter, MultiMetric, Runner, Provider} from '@angular/benchpress'; +import {readCommandLine as readE2eCommandLine, openBrowser} from './e2e_util'; + +let cmdArgs: {'sample-size': number, 'force-gc': boolean, 'dryrun': boolean, 'bundles': boolean}; +let runner: Runner; + +export function readCommandLine() { + cmdArgs = readE2eCommandLine({ + 'sample-size': {describe: 'Used for perf: sample size.', default: 20}, + 'force-gc': {describe: 'Used for perf: force gc.', default: false, type: 'boolean'}, + 'dryrun': {describe: 'If true, only run performance benchmarks once.', default: false}, + 'bundles': {describe: 'Whether to use the angular bundles or not.', default: false} + }); + runner = createBenchpressRunner(); } -export function runBenchmark(config: any /** TODO #9100 */) { - return getScaleFactor(browser.params.benchmark.scaling).then(function(scaleFactor) { - var description = {}; - var urlParams: any[] /** TODO #9100 */ = []; - if (config.params) { - config.params.forEach(function(param: any /** TODO #9100 */) { - var name = param.name; - var value = applyScaleFactor(param.value, scaleFactor, param.scale); - urlParams.push(name + '=' + value); - (description as any /** TODO #9100 */)[name] = value; - }); - } - var url = encodeURI(config.url + '?' + urlParams.join('&')); - return browser.get(url).then(function() { - return (global as any /** TODO #9100 */)['benchpressRunner'].sample({ - id: config.id, - execute: config.work, - prepare: config.prepare, - microMetrics: config.microMetrics, - providers: [{provide: Options.SAMPLE_DESCRIPTION, useValue: description}] - }); - }); +export function runBenchmark(config: { + id: string, + url: string, + params: {name: string, value: any}[], + ignoreBrowserSynchronization?: boolean, + microMetrics?: {[key: string]: string}, + work?: () => void, + prepare?: () => void, +}): Promise { + openBrowser(config); + + var description: {[key: string]: any} = {'bundles': cmdArgs.bundles}; + config.params.forEach((param) => { description[param.name] = param.value; }); + return runner.sample({ + id: config.id, + execute: config.work, + prepare: config.prepare, + microMetrics: config.microMetrics, + providers: [{provide: Options.SAMPLE_DESCRIPTION, useValue: description}] }); } -function getScaleFactor(possibleScalings: any /** TODO #9100 */) { - return browser.executeScript('return navigator.userAgent').then(function(userAgent: string) { - var scaleFactor = 1; - possibleScalings.forEach(function(entry: any /** TODO #9100 */) { - if (userAgent.match(entry.userAgent)) { - scaleFactor = entry.value; - } - }); - return scaleFactor; - }); -} - -function applyScaleFactor( - value: any /** TODO #9100 */, scaleFactor: any /** TODO #9100 */, - method: any /** TODO #9100 */) { - if (method === 'log2') { - return value + Math.log(scaleFactor) / Math.LN2; - } else if (method === 'sqrt') { - return value * Math.sqrt(scaleFactor); - } else if (method === 'linear') { - return value * scaleFactor; - } else { - return value; +function createBenchpressRunner(): Runner { + let runId = nodeUuid.v1(); + if (process.env.GIT_SHA) { + runId = process.env.GIT_SHA + ' ' + runId; } + const resultsFolder = './dist/benchmark_results'; + fs.ensureDirSync(resultsFolder); + let providers: Provider[] = [ + SeleniumWebDriverAdapter.PROTRACTOR_PROVIDERS, + {provide: Options.FORCE_GC, useValue: cmdArgs['force-gc']}, + {provide: Options.DEFAULT_DESCRIPTION, useValue: {'runId': runId}}, JsonFileReporter.PROVIDERS, + {provide: JsonFileReporter.PATH, useValue: resultsFolder} + ]; + if (!cmdArgs['dryrun']) { + providers.push({provide: Validator, useExisting: RegressionSlopeValidator}); + providers.push( + {provide: RegressionSlopeValidator.SAMPLE_SIZE, useValue: cmdArgs['sample-size']}); + providers.push(MultiReporter.provideWith([ConsoleReporter, JsonFileReporter])); + } else { + providers.push({provide: Validator, useExisting: SizeValidator}); + providers.push({provide: SizeValidator.SAMPLE_SIZE, useValue: 1}); + providers.push(MultiReporter.provideWith([])); + providers.push(MultiMetric.provideWith([])); + } + return new Runner(providers); } diff --git a/protractor-e2e.conf.js b/protractor-e2e.conf.js index d1ec5c1165..db3707f30c 100644 --- a/protractor-e2e.conf.js +++ b/protractor-e2e.conf.js @@ -1,3 +1,7 @@ +// Make sure that the command line is read as the first thing +// as this could exit node if the help script should be printed. +require('./dist/all/e2e_util/e2e_util').readCommandLine(); + var BROWSER_OPTIONS = { LocalChrome: { 'browserName': 'chrome' @@ -11,8 +15,6 @@ var BROWSER_OPTIONS = { } }; - - exports.config = { allScriptsTimeout: 11000, specs: [ diff --git a/protractor-perf.conf.js b/protractor-perf.conf.js new file mode 100644 index 0000000000..9b4c5d60e5 --- /dev/null +++ b/protractor-perf.conf.js @@ -0,0 +1,57 @@ +// Make sure that the command line is read as the first thing +// as this could exit node if the help script should be printed. +require('./dist/all/e2e_util/perf_util').readCommandLine(); + +var CHROME_OPTIONS = { + 'args': ['--js-flags=--expose-gc'], + 'perfLoggingPrefs': { + 'traceCategories': 'v8,blink.console,devtools.timeline,disabled-by-default-devtools.timeline' + } +}; + +var BROWSER_CAPS = { + LocalChrome: { + 'browserName': 'chrome', + chromeOptions: CHROME_OPTIONS, + loggingPrefs: { + performance: 'ALL', + browser: 'ALL' + } + }, + ChromeOnTravis: { + browserName: 'chrome', + chromeOptions: mergeInto(CHROME_OPTIONS, { + 'args': ['--no-sandbox'], + 'binary': process.env.CHROME_BIN + }), + loggingPrefs: { + performance: 'ALL', + browser: 'ALL' + } + } +}; + +exports.config = { + restartBrowserBetweenTests: true, + allScriptsTimeout: 11000, + specs: [ + 'dist/all/**/e2e_test/**/*_perf.js' + ], + capabilities: process.env.TRAVIS ? BROWSER_CAPS.ChromeOnTravis : BROWSER_CAPS.LocalChrome, + directConnect: true, + baseUrl: 'http://localhost:8000/', + framework: 'jasmine2', + jasmineNodeOpts: { + showColors: true, + defaultTimeoutInterval: 60000, + print: function(msg) { console.log(msg)} + }, + useAllAngular2AppRoots: true +}; + +function mergeInto(src, target) { +for (var prop in src) { + target[prop] = src[prop]; +} +return target; +} diff --git a/scripts/ci-lite/build.sh b/scripts/ci-lite/build.sh index 8ea2e96d89..bceb7e2c5d 100755 --- a/scripts/ci-lite/build.sh +++ b/scripts/ci-lite/build.sh @@ -16,6 +16,5 @@ node dist/tools/@angular/tsc-wrapped/src/main -p modules node dist/tools/@angular/tsc-wrapped/src/main -p modules/@angular/core node dist/tools/@angular/tsc-wrapped/src/main -p modules/@angular/common node dist/tools/@angular/tsc-wrapped/src/main -p modules/@angular/router -$(npm bin)/tsc -p modules/benchpress echo 'travis_fold:end:BUILD' diff --git a/scripts/ci-lite/test_e2e.sh b/scripts/ci-lite/test_e2e.sh index 1c2618c28a..0f3edeb143 100755 --- a/scripts/ci-lite/test_e2e.sh +++ b/scripts/ci-lite/test_e2e.sh @@ -35,7 +35,8 @@ cd .. if [[ ${TRAVIS} ]]; then sh -e /etc/init.d/xvfb start fi -NODE_PATH=$NODE_PATH:./dist/all $(npm bin)/protractor ./protractor-e2e.conf.js +NODE_PATH=$NODE_PATH:./dist/all $(npm bin)/protractor ./protractor-e2e.conf.js --bundles=true +NODE_PATH=$NODE_PATH:./dist/all $(npm bin)/protractor ./protractor-perf.conf.js --bundles=true --dryrun echo 'travis_fold:end:test.e2e.localChrome' echo 'travis_fold:end:test.js' diff --git a/tools/tsc-watch/index.ts b/tools/tsc-watch/index.ts index 856ca1a922..3a0a107711 100644 --- a/tools/tsc-watch/index.ts +++ b/tools/tsc-watch/index.ts @@ -72,8 +72,7 @@ if (platform == 'node') { processOutputEmitterCodeGen, [ 'node', 'dist/tools/cjs-jasmine', '--', '@angular/**/*_spec.js', - '@angular/compiler-cli/test/**/*_spec.js', - '@angular/benchpress/test/**/*_spec.js' + '@angular/compiler-cli/test/**/*_spec.js', '@angular/benchpress/test/**/*_spec.js' ] ] },