fix(build): simplify the e2e/perf config via command line arguments

README.md

@@ -54,12 +54,25 @@ the transpiler is reloaded. With that it is possible to make changes
 to the preprocessor and run the tests without exiting karma
 (just touch a test file that you would like to run).
 
+### E2e tests
+
+1. `gulp build.js.cjs` (builds benchpress and tests into `dist/js/cjs` folder)
+2. `gulp serve.js.prod serve.js.dart2js` (runs local webserver)
+3. `protractor protractor-js.conf.js`: JS e2e tests
+4. `protractor protractor-dart2js.conf.js`: Dart2JS e2e tests
+
+Angular specific command line options when running protractor:
+`protractor protractor-{js|dart2js}-conf.js --ng-help`
+
 ### Performance tests
 
 1. `gulp build.js.cjs` (builds benchpress and tests into `dist/js/cjs` folder)
 2. `gulp serve.js.prod serve.js.dart2js` (runs local webserver)
-3. `protractor protractor-perf-js.conf.js`: JS performance tests
+3. `protractor protractor-js.conf.js --benchmark`: JS performance tests
-4. `protractor protractor-perf-dart2js.conf.js`: Dart2JS performance tests
+4. `protractor protractor-dart2js.conf.js --benchmark`: Dart2JS performance tests
+
+Angular specific command line options when running protractor (e.g. force gc, ...):
+`protractor protractor-{js|dart2js}-conf.js --ng-help`
 
 ### Examples
 

protractor-dart2js.conf.js

@@ -0,0 +1,13 @@
+var data = module.exports = require('./protractor-shared.js');
+var config = data.config;
+
+config.baseUrl = 'http://localhost:8002/';
+
+config.exclude.push(
+  'dist/js/cjs/examples/e2e_test/sourcemap/sourcemap_spec.js',
+  // TODO: remove this line when largetable dart has been added
+  'dist/js/cjs/benchmarks_external/e2e_test/largetable_perf.js'
+);
+
+data.createBenchpressRunner({ lang: 'dart' });
+

protractor-e2e-dart2js.conf.js

@@ -1,16 +0,0 @@
-var data = module.exports = require('./protractor-e2e-shared.js');
-var config = data.config;
-
-config.baseUrl = 'http://localhost:8002/';
-
-// TODO: remove this line when largetable dart has been added
-config.exclude = config.exclude || [];
-config.exclude.push('dist/js/cjs/examples/e2e_test/sourcemap/sourcemap_spec.js');
-config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/largetable_perf.js');
-
-data.createBenchpressRunner({
-  forceGc: false,
-  lang: 'dart',
-  test: true,
-  sampleSize: 1
-});

protractor-e2e-shared.js

@@ -1,5 +0,0 @@
-var data = module.exports = require('./protractor-shared.js');
-var config = data.config;
-
-config.specs = ['dist/js/cjs/**/e2e_test/**/*_spec.js', 'dist/js/cjs/**/e2e_test/**/*_perf.js'];
-config.exclude = ['dist/js/cjs/**/node_modules/**'];

protractor-js.conf.js

@@ -1,16 +1,10 @@
-var data = module.exports = require('./protractor-e2e-shared.js');
+var data = module.exports = require('./protractor-shared.js');
 var config = data.config;
 
 config.baseUrl = 'http://localhost:8001/';
-
 // TODO: remove exclusion when JS verison of scrolling benchmark is available
-config.exclude = config.exclude || [];
+config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/naive_infinite_scroll_spec.js');
 config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/naive_infinite_scroll_perf.js');
 
-data.createBenchpressRunner({
+data.createBenchpressRunner({ lang: 'js' });
-  forceGc: false,
-  lang: 'js',
-  test: true,
-  sampleSize: 1
-});
 

protractor-perf-dart2js.conf.js

@@ -1,16 +0,0 @@
-var data = module.exports = require('./protractor-perf-shared.js');
-var config = data.config;
-
-config.baseUrl = 'http://localhost:8002/';
-
-// TODO: remove this line when largetable dart has been added
-config.exclude = config.exclude || [];
-config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/largetable_perf.js');
-
-data.createBenchpressRunner({
-  forceGc: false,
-  lang: 'dart',
-  test: false,
-  sampleSize: 20
-});
-

protractor-perf-js.conf.js

@@ -1,16 +0,0 @@
-var data = module.exports = require('./protractor-perf-shared.js');
-var config = data.config;
-
-config.baseUrl = 'http://localhost:8001/';
-
-// TODO: remove exclusion when JS verison of scrolling benchmark is available
-config.exclude = config.exclude || [];
-config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/naive_infinite_scroll_perf.js');
-
-data.createBenchpressRunner({
-  forceGc: false,
-  lang: 'js',
-  test: false,
-  sampleSize: 20
-});
-

protractor-perf-shared.js

@@ -1,8 +0,0 @@
-var data = module.exports = require('./protractor-shared.js');
-var config = data.config;
-
-config.specs = ['dist/js/cjs/**/e2e_test/**/*_perf.js'];
-config.exclude = ['dist/js/cjs/**/node_modules/**'];
-
-config.jasmineNodeOpts.defaultTimeoutInterval = 80000;
-

protractor-shared.js

@@ -1,83 +1,35 @@
 // load traceur runtime as our tests are written in es6
 require('traceur/bin/traceur-runtime.js');
 
-var nodeUuid = require('node-uuid');
+var argv = require('yargs')
-var benchpress = require('./dist/js/cjs/benchpress/benchpress');
+    .usage('Angular e2e/perf test options.')
-var SeleniumWebDriverAdapter = require('./dist/js/cjs/benchpress/src/webdriver/selenium_webdriver_adapter').SeleniumWebDriverAdapter;
+    .options({
-var cmdArgs = require('minimist')(process.argv);
+      'sample-size': {
-
+        describe: 'sample size',
-var cmdLineBrowsers = cmdArgs.browsers ? cmdArgs.browsers.split(',') : [];
+        default: 20,
-
+        type: 'boolean'
-var config = exports.config = {
+      },
-  // Disable waiting for Angular as we don't have an integration layer yet...
+      'force-gc': {
-  // TODO(tbosch): Implement a proper debugging API for Ng2.0, remove this here
+        describe: 'force gc',
-  // and the sleeps in all tests.
+        default: false,
-  onPrepare: function() {
+        type: 'boolean'
-    browser.ignoreSynchronization = true;
+      },
-    var _get = browser.get;
+      'benchmark': {
-    var sleepInterval = process.env.TRAVIS || process.env.JENKINS_URL ? 7000 : 3000;
+        describe: 'whether to run the benchmarks',
-    browser.get = function() {
+        default: false
-      var result = _get.apply(this, arguments);
+      },
-      browser.sleep(sleepInterval);
+      'browsers': {
-      return result;
+        describe: 'preconfigured browsers that should be used',
-    }
+        default: 'ChromeDesktop'
-  },
+      }
-
-  framework: 'jasmine2',
-
-  jasmineNodeOpts: {
-    showColors: true,
-    defaultTimeoutInterval: 30000
-  },
-  params: {
-    benchmark: {
-      scaling: [{
-        userAgent: /Android/, value: 0.125
-      }]
-    }
-  }
-};
-
-exports.createBenchpressRunner = function(options) {
-  // TODO(tbosch): add cloud reporter again (only when !options.test)
-  // var cloudReporterConfig;
-  // if (process.env.CLOUD_SECRET_PATH) {
-  //   console.log('using cloud reporter!');
-  //   cloudReporterConfig = {
-  //     auth: require(process.env.CLOUD_SECRET_PATH),
-  //     projectId: 'angular-perf',
-  //     datasetId: 'benchmarks',
-  //     tableId: 'ng2perf'
-  //   };
-  // }
-
-  var runId = nodeUuid.v1();
-  if (process.env.GIT_SHA) {
-    runId = process.env.GIT_SHA + ' ' + runId;
-  }
-  var bindings = [
-    benchpress.bind(benchpress.WebDriverAdapter).toFactory(
-      function() { return new SeleniumWebDriverAdapter(global.browser); }, []
-    ),
-    benchpress.bind(benchpress.Options.FORCE_GC).toValue(options.forceGc),
-    benchpress.bind(benchpress.Options.DEFAULT_DESCRIPTION).toValue({
-      'lang': options.lang,
-      'runId': runId
     })
-  ];
+    .help('ng-help')
-  if (options.test) {
+    .wrap(40)
-    bindings.push(benchpress.SizeValidator.BINDINGS);
+    .argv
-    bindings.push(benchpress.bind(benchpress.SizeValidator.SAMPLE_SIZE).toValue(1));
-  } else {
-    bindings.push(benchpress.RegressionSlopeValidator.BINDINGS);
-    bindings.push(benchpress.bind(benchpress.RegressionSlopeValidator.SAMPLE_SIZE).toValue(options.sampleSize));
-  }
 
-  global.benchpressRunner = new benchpress.Runner(bindings);
+var browsers = argv['browsers'].split(',');
-}
 
-
+var BROWSER_CAPS = {
-var POSSIBLE_CAPS = {
   Dartium: {
     name: 'Dartium',
     browserName: 'chrome',
@@ -112,14 +64,96 @@ var POSSIBLE_CAPS = {
     }
   }
 };
-if (cmdLineBrowsers.length) {
+
-  config.multiCapabilities = cmdLineBrowsers.map(function(browserName) {
+var config = exports.config = {
-    var caps = POSSIBLE_CAPS[browserName];
+  // Disable waiting for Angular as we don't have an integration layer yet...
+  // TODO(tbosch): Implement a proper debugging API for Ng2.0, remove this here
+  // and the sleeps in all tests.
+  onPrepare: function() {
+    browser.ignoreSynchronization = true;
+    var _get = browser.get;
+    var sleepInterval = process.env.TRAVIS || process.env.JENKINS_URL ? 7000 : 3000;
+    browser.get = function() {
+      var result = _get.apply(this, arguments);
+      browser.sleep(sleepInterval);
+      return result;
+    }
+  },
+
+  specs: argv['benchmark'] ? [
+    'dist/js/cjs/**/e2e_test/**/*_perf.js'
+  ] : [
+    'dist/js/cjs/**/e2e_test/**/*_spec.js',
+    'dist/js/cjs/**/e2e_test/**/*_perf.js'
+  ],
+
+  exclude: [
+    'dist/js/cjs/**/node_modules/**',
+  ],
+
+  multiCapabilities: browsers.map(function(browserName) {
+    var caps = BROWSER_CAPS[browserName];
+    console.log('Testing against', browserName);
     if (!caps) {
       throw new Error('Not configured browser name: '+browserName);
     }
     return caps;
-  });
+  }),
-} else {
+
-  config.multiCapabilities = [POSSIBLE_CAPS.ChromeDesktop];
+  framework: 'jasmine2',
+
+  jasmineNodeOpts: {
+    showColors: true,
+    defaultTimeoutInterval: argv.benchpress ? 80000 : 30000
+  },
+  params: {
+    benchmark: {
+      scaling: [{
+        userAgent: /Android/, value: 0.125
+      }]
+    }
+  }
+};
+
+exports.createBenchpressRunner = function(options) {
+  var nodeUuid = require('node-uuid');
+  var benchpress = require('./dist/js/cjs/benchpress/benchpress');
+  var SeleniumWebDriverAdapter =
+    require('./dist/js/cjs/benchpress/src/webdriver/selenium_webdriver_adapter').SeleniumWebDriverAdapter;
+
+  // TODO(tbosch): add cloud reporter again (only when !options.test)
+  // var cloudReporterConfig;
+  // if (process.env.CLOUD_SECRET_PATH) {
+  //   console.log('using cloud reporter!');
+  //   cloudReporterConfig = {
+  //     auth: require(process.env.CLOUD_SECRET_PATH),
+  //     projectId: 'angular-perf',
+  //     datasetId: 'benchmarks',
+  //     tableId: 'ng2perf'
+  //   };
+  // }
+
+  var runId = nodeUuid.v1();
+  if (process.env.GIT_SHA) {
+    runId = process.env.GIT_SHA + ' ' + runId;
+  }
+  var bindings = [
+    benchpress.bind(benchpress.WebDriverAdapter).toFactory(
+      function() { return new SeleniumWebDriverAdapter(global.browser); }, []
+    ),
+    benchpress.bind(benchpress.Options.FORCE_GC).toValue(argv['force-gc']),
+    benchpress.bind(benchpress.Options.DEFAULT_DESCRIPTION).toValue({
+      'lang': options.lang,
+      'runId': runId
+    })
+  ];
+  if (argv['benchmark']) {
+    bindings.push(benchpress.RegressionSlopeValidator.BINDINGS);
+    bindings.push(benchpress.bind(benchpress.RegressionSlopeValidator.SAMPLE_SIZE).toValue(argv['sample-size']));
+  } else {
+    bindings.push(benchpress.SizeValidator.BINDINGS);
+    bindings.push(benchpress.bind(benchpress.SizeValidator.SAMPLE_SIZE).toValue(1));
+  }
+
+  global.benchpressRunner = new benchpress.Runner(bindings);
 }

scripts/ci/test_e2e_dart.sh

@@ -21,4 +21,4 @@ trap killServer EXIT
 # wait for server to come up!
 sleep 10
 
-./node_modules/.bin/protractor protractor-e2e-dart2js.conf.js --browsers=$E2E_BROWSERS
+./node_modules/.bin/protractor protractor-dart2js.conf.js --browsers=$E2E_BROWSERS

scripts/ci/test_e2e_js.sh

@@ -21,4 +21,4 @@ trap killServer EXIT
 # wait for server to come up!
 sleep 10
 
-./node_modules/.bin/protractor protractor-e2e-js.conf.js --browsers=$E2E_BROWSERS
+./node_modules/.bin/protractor protractor-js.conf.js --browsers=$E2E_BROWSERS

scripts/ci/test_perf.sh

@@ -21,5 +21,5 @@ trap killServer EXIT
 # wait for server to come up!
 sleep 10
 
-./node_modules/.bin/protractor protractor-perf-js.conf.js --browsers=$PERF_BROWSERS
+./node_modules/.bin/protractor protractor-js.conf.js --browsers=$PERF_BROWSERS --benchmark
-./node_modules/.bin/protractor protractor-perf-dart2js.conf.js --browsers=$PERF_BROWSERS
+./node_modules/.bin/protractor protractor-dart2js.conf.js --browsers=$PERF_BROWSERS --benchmark