diff --git a/README.md b/README.md index 1a98e31f4a..8f36f37241 100644 --- a/README.md +++ b/README.md @@ -54,12 +54,25 @@ the transpiler is reloaded. With that it is possible to make changes to the preprocessor and run the tests without exiting karma (just touch a test file that you would like to run). +### E2e tests + +1. `gulp build.js.cjs` (builds benchpress and tests into `dist/js/cjs` folder) +2. `gulp serve.js.prod serve.js.dart2js` (runs local webserver) +3. `protractor protractor-js.conf.js`: JS e2e tests +4. `protractor protractor-dart2js.conf.js`: Dart2JS e2e tests + +Angular specific command line options when running protractor: +`protractor protractor-{js|dart2js}-conf.js --ng-help` + ### Performance tests 1. `gulp build.js.cjs` (builds benchpress and tests into `dist/js/cjs` folder) 2. `gulp serve.js.prod serve.js.dart2js` (runs local webserver) -3. `protractor protractor-perf-js.conf.js`: JS performance tests -4. `protractor protractor-perf-dart2js.conf.js`: Dart2JS performance tests +3. `protractor protractor-js.conf.js --benchmark`: JS performance tests +4. `protractor protractor-dart2js.conf.js --benchmark`: Dart2JS performance tests + +Angular specific command line options when running protractor (e.g. force gc, ...): +`protractor protractor-{js|dart2js}-conf.js --ng-help` ### Examples diff --git a/protractor-dart2js.conf.js b/protractor-dart2js.conf.js new file mode 100644 index 0000000000..53066b7b4a --- /dev/null +++ b/protractor-dart2js.conf.js @@ -0,0 +1,13 @@ +var data = module.exports = require('./protractor-shared.js'); +var config = data.config; + +config.baseUrl = 'http://localhost:8002/'; + +config.exclude.push( + 'dist/js/cjs/examples/e2e_test/sourcemap/sourcemap_spec.js', + // TODO: remove this line when largetable dart has been added + 'dist/js/cjs/benchmarks_external/e2e_test/largetable_perf.js' +); + +data.createBenchpressRunner({ lang: 'dart' }); + diff --git a/protractor-e2e-dart2js.conf.js b/protractor-e2e-dart2js.conf.js deleted file mode 100644 index 9f389e8e13..0000000000 --- a/protractor-e2e-dart2js.conf.js +++ /dev/null @@ -1,16 +0,0 @@ -var data = module.exports = require('./protractor-e2e-shared.js'); -var config = data.config; - -config.baseUrl = 'http://localhost:8002/'; - -// TODO: remove this line when largetable dart has been added -config.exclude = config.exclude || []; -config.exclude.push('dist/js/cjs/examples/e2e_test/sourcemap/sourcemap_spec.js'); -config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/largetable_perf.js'); - -data.createBenchpressRunner({ - forceGc: false, - lang: 'dart', - test: true, - sampleSize: 1 -}); diff --git a/protractor-e2e-shared.js b/protractor-e2e-shared.js deleted file mode 100644 index dabc5cbecd..0000000000 --- a/protractor-e2e-shared.js +++ /dev/null @@ -1,5 +0,0 @@ -var data = module.exports = require('./protractor-shared.js'); -var config = data.config; - -config.specs = ['dist/js/cjs/**/e2e_test/**/*_spec.js', 'dist/js/cjs/**/e2e_test/**/*_perf.js']; -config.exclude = ['dist/js/cjs/**/node_modules/**']; diff --git a/protractor-e2e-js.conf.js b/protractor-js.conf.js similarity index 54% rename from protractor-e2e-js.conf.js rename to protractor-js.conf.js index 3b31799aeb..b032bfad2f 100644 --- a/protractor-e2e-js.conf.js +++ b/protractor-js.conf.js @@ -1,16 +1,10 @@ -var data = module.exports = require('./protractor-e2e-shared.js'); +var data = module.exports = require('./protractor-shared.js'); var config = data.config; config.baseUrl = 'http://localhost:8001/'; - // TODO: remove exclusion when JS verison of scrolling benchmark is available -config.exclude = config.exclude || []; +config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/naive_infinite_scroll_spec.js'); config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/naive_infinite_scroll_perf.js'); -data.createBenchpressRunner({ - forceGc: false, - lang: 'js', - test: true, - sampleSize: 1 -}); +data.createBenchpressRunner({ lang: 'js' }); diff --git a/protractor-perf-dart2js.conf.js b/protractor-perf-dart2js.conf.js deleted file mode 100644 index 09b89ed1bd..0000000000 --- a/protractor-perf-dart2js.conf.js +++ /dev/null @@ -1,16 +0,0 @@ -var data = module.exports = require('./protractor-perf-shared.js'); -var config = data.config; - -config.baseUrl = 'http://localhost:8002/'; - -// TODO: remove this line when largetable dart has been added -config.exclude = config.exclude || []; -config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/largetable_perf.js'); - -data.createBenchpressRunner({ - forceGc: false, - lang: 'dart', - test: false, - sampleSize: 20 -}); - diff --git a/protractor-perf-js.conf.js b/protractor-perf-js.conf.js deleted file mode 100644 index 2d6250808f..0000000000 --- a/protractor-perf-js.conf.js +++ /dev/null @@ -1,16 +0,0 @@ -var data = module.exports = require('./protractor-perf-shared.js'); -var config = data.config; - -config.baseUrl = 'http://localhost:8001/'; - -// TODO: remove exclusion when JS verison of scrolling benchmark is available -config.exclude = config.exclude || []; -config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/naive_infinite_scroll_perf.js'); - -data.createBenchpressRunner({ - forceGc: false, - lang: 'js', - test: false, - sampleSize: 20 -}); - diff --git a/protractor-perf-shared.js b/protractor-perf-shared.js deleted file mode 100644 index 2dea440068..0000000000 --- a/protractor-perf-shared.js +++ /dev/null @@ -1,8 +0,0 @@ -var data = module.exports = require('./protractor-shared.js'); -var config = data.config; - -config.specs = ['dist/js/cjs/**/e2e_test/**/*_perf.js']; -config.exclude = ['dist/js/cjs/**/node_modules/**']; - -config.jasmineNodeOpts.defaultTimeoutInterval = 80000; - diff --git a/protractor-shared.js b/protractor-shared.js index 6ac6740220..54b07fd57c 100644 --- a/protractor-shared.js +++ b/protractor-shared.js @@ -1,83 +1,35 @@ // load traceur runtime as our tests are written in es6 require('traceur/bin/traceur-runtime.js'); -var nodeUuid = require('node-uuid'); -var benchpress = require('./dist/js/cjs/benchpress/benchpress'); -var SeleniumWebDriverAdapter = require('./dist/js/cjs/benchpress/src/webdriver/selenium_webdriver_adapter').SeleniumWebDriverAdapter; -var cmdArgs = require('minimist')(process.argv); - -var cmdLineBrowsers = cmdArgs.browsers ? cmdArgs.browsers.split(',') : []; - -var config = exports.config = { - // Disable waiting for Angular as we don't have an integration layer yet... - // TODO(tbosch): Implement a proper debugging API for Ng2.0, remove this here - // and the sleeps in all tests. - onPrepare: function() { - browser.ignoreSynchronization = true; - var _get = browser.get; - var sleepInterval = process.env.TRAVIS || process.env.JENKINS_URL ? 7000 : 3000; - browser.get = function() { - var result = _get.apply(this, arguments); - browser.sleep(sleepInterval); - return result; - } - }, - - framework: 'jasmine2', - - jasmineNodeOpts: { - showColors: true, - defaultTimeoutInterval: 30000 - }, - params: { - benchmark: { - scaling: [{ - userAgent: /Android/, value: 0.125 - }] - } - } -}; - -exports.createBenchpressRunner = function(options) { - // TODO(tbosch): add cloud reporter again (only when !options.test) - // var cloudReporterConfig; - // if (process.env.CLOUD_SECRET_PATH) { - // console.log('using cloud reporter!'); - // cloudReporterConfig = { - // auth: require(process.env.CLOUD_SECRET_PATH), - // projectId: 'angular-perf', - // datasetId: 'benchmarks', - // tableId: 'ng2perf' - // }; - // } - - var runId = nodeUuid.v1(); - if (process.env.GIT_SHA) { - runId = process.env.GIT_SHA + ' ' + runId; - } - var bindings = [ - benchpress.bind(benchpress.WebDriverAdapter).toFactory( - function() { return new SeleniumWebDriverAdapter(global.browser); }, [] - ), - benchpress.bind(benchpress.Options.FORCE_GC).toValue(options.forceGc), - benchpress.bind(benchpress.Options.DEFAULT_DESCRIPTION).toValue({ - 'lang': options.lang, - 'runId': runId +var argv = require('yargs') + .usage('Angular e2e/perf test options.') + .options({ + 'sample-size': { + describe: 'sample size', + default: 20, + type: 'boolean' + }, + 'force-gc': { + describe: 'force gc', + default: false, + type: 'boolean' + }, + 'benchmark': { + describe: 'whether to run the benchmarks', + default: false + }, + 'browsers': { + describe: 'preconfigured browsers that should be used', + default: 'ChromeDesktop' + } }) - ]; - if (options.test) { - bindings.push(benchpress.SizeValidator.BINDINGS); - bindings.push(benchpress.bind(benchpress.SizeValidator.SAMPLE_SIZE).toValue(1)); - } else { - bindings.push(benchpress.RegressionSlopeValidator.BINDINGS); - bindings.push(benchpress.bind(benchpress.RegressionSlopeValidator.SAMPLE_SIZE).toValue(options.sampleSize)); - } + .help('ng-help') + .wrap(40) + .argv - global.benchpressRunner = new benchpress.Runner(bindings); -} +var browsers = argv['browsers'].split(','); - -var POSSIBLE_CAPS = { +var BROWSER_CAPS = { Dartium: { name: 'Dartium', browserName: 'chrome', @@ -112,14 +64,96 @@ var POSSIBLE_CAPS = { } } }; -if (cmdLineBrowsers.length) { - config.multiCapabilities = cmdLineBrowsers.map(function(browserName) { - var caps = POSSIBLE_CAPS[browserName]; + +var config = exports.config = { + // Disable waiting for Angular as we don't have an integration layer yet... + // TODO(tbosch): Implement a proper debugging API for Ng2.0, remove this here + // and the sleeps in all tests. + onPrepare: function() { + browser.ignoreSynchronization = true; + var _get = browser.get; + var sleepInterval = process.env.TRAVIS || process.env.JENKINS_URL ? 7000 : 3000; + browser.get = function() { + var result = _get.apply(this, arguments); + browser.sleep(sleepInterval); + return result; + } + }, + + specs: argv['benchmark'] ? [ + 'dist/js/cjs/**/e2e_test/**/*_perf.js' + ] : [ + 'dist/js/cjs/**/e2e_test/**/*_spec.js', + 'dist/js/cjs/**/e2e_test/**/*_perf.js' + ], + + exclude: [ + 'dist/js/cjs/**/node_modules/**', + ], + + multiCapabilities: browsers.map(function(browserName) { + var caps = BROWSER_CAPS[browserName]; + console.log('Testing against', browserName); if (!caps) { throw new Error('Not configured browser name: '+browserName); } return caps; - }); -} else { - config.multiCapabilities = [POSSIBLE_CAPS.ChromeDesktop]; + }), + + framework: 'jasmine2', + + jasmineNodeOpts: { + showColors: true, + defaultTimeoutInterval: argv.benchpress ? 80000 : 30000 + }, + params: { + benchmark: { + scaling: [{ + userAgent: /Android/, value: 0.125 + }] + } + } +}; + +exports.createBenchpressRunner = function(options) { + var nodeUuid = require('node-uuid'); + var benchpress = require('./dist/js/cjs/benchpress/benchpress'); + var SeleniumWebDriverAdapter = + require('./dist/js/cjs/benchpress/src/webdriver/selenium_webdriver_adapter').SeleniumWebDriverAdapter; + + // TODO(tbosch): add cloud reporter again (only when !options.test) + // var cloudReporterConfig; + // if (process.env.CLOUD_SECRET_PATH) { + // console.log('using cloud reporter!'); + // cloudReporterConfig = { + // auth: require(process.env.CLOUD_SECRET_PATH), + // projectId: 'angular-perf', + // datasetId: 'benchmarks', + // tableId: 'ng2perf' + // }; + // } + + var runId = nodeUuid.v1(); + if (process.env.GIT_SHA) { + runId = process.env.GIT_SHA + ' ' + runId; + } + var bindings = [ + benchpress.bind(benchpress.WebDriverAdapter).toFactory( + function() { return new SeleniumWebDriverAdapter(global.browser); }, [] + ), + benchpress.bind(benchpress.Options.FORCE_GC).toValue(argv['force-gc']), + benchpress.bind(benchpress.Options.DEFAULT_DESCRIPTION).toValue({ + 'lang': options.lang, + 'runId': runId + }) + ]; + if (argv['benchmark']) { + bindings.push(benchpress.RegressionSlopeValidator.BINDINGS); + bindings.push(benchpress.bind(benchpress.RegressionSlopeValidator.SAMPLE_SIZE).toValue(argv['sample-size'])); + } else { + bindings.push(benchpress.SizeValidator.BINDINGS); + bindings.push(benchpress.bind(benchpress.SizeValidator.SAMPLE_SIZE).toValue(1)); + } + + global.benchpressRunner = new benchpress.Runner(bindings); } diff --git a/scripts/ci/test_e2e_dart.sh b/scripts/ci/test_e2e_dart.sh index f97c16d5f9..cad86c1049 100755 --- a/scripts/ci/test_e2e_dart.sh +++ b/scripts/ci/test_e2e_dart.sh @@ -21,4 +21,4 @@ trap killServer EXIT # wait for server to come up! sleep 10 -./node_modules/.bin/protractor protractor-e2e-dart2js.conf.js --browsers=$E2E_BROWSERS \ No newline at end of file +./node_modules/.bin/protractor protractor-dart2js.conf.js --browsers=$E2E_BROWSERS \ No newline at end of file diff --git a/scripts/ci/test_e2e_js.sh b/scripts/ci/test_e2e_js.sh index 59b1eec99f..3e151ff73d 100755 --- a/scripts/ci/test_e2e_js.sh +++ b/scripts/ci/test_e2e_js.sh @@ -21,4 +21,4 @@ trap killServer EXIT # wait for server to come up! sleep 10 -./node_modules/.bin/protractor protractor-e2e-js.conf.js --browsers=$E2E_BROWSERS +./node_modules/.bin/protractor protractor-js.conf.js --browsers=$E2E_BROWSERS diff --git a/scripts/ci/test_perf.sh b/scripts/ci/test_perf.sh index 78ed851fd2..6fc249c945 100755 --- a/scripts/ci/test_perf.sh +++ b/scripts/ci/test_perf.sh @@ -21,5 +21,5 @@ trap killServer EXIT # wait for server to come up! sleep 10 -./node_modules/.bin/protractor protractor-perf-js.conf.js --browsers=$PERF_BROWSERS -./node_modules/.bin/protractor protractor-perf-dart2js.conf.js --browsers=$PERF_BROWSERS \ No newline at end of file +./node_modules/.bin/protractor protractor-js.conf.js --browsers=$PERF_BROWSERS --benchmark +./node_modules/.bin/protractor protractor-dart2js.conf.js --browsers=$PERF_BROWSERS --benchmark \ No newline at end of file