refactor(build): support multiple configurable browsers for e2e and perf tests

Also includes browser configuration for Android

.travis.yml

@@ -5,17 +5,23 @@ node_js:
 env:
   global:
   - CHANNEL=stable
-  - BROWSERS=DartiumWithWebPlatform
+  - KARMA_BROWSERS=DartiumWithWebPlatform
+  - E2E_BROWSERS=Dartium
   - LOGS_DIR=/tmp/angular-build/logs
+  - ARCH=linux-x64
 
 before_install:
 - export DISPLAY=:99.0
-- ./scripts/travis/install.sh
+- export GIT_SHA=$(git rev-parse HEAD)
+- ./scripts/ci/init_android.sh
+- ./scripts/ci/install_dart.sh
 - sh -e /etc/init.d/xvfb start
 - if [[ -e SKIP_TRAVIS_TESTS ]]; then { cat SKIP_TRAVIS_TESTS ; exit 0; } fi
 before_script:
 - mkdir -p $LOGS_DIR
 script:
-- ./scripts/travis/build.sh
+- ./scripts/ci/build.sh
+- ./scripts/ci/test_unit.sh
+- ./scripts/ci/test_e2e.sh
 after_script:
-- ./scripts/travis/print-logs.sh
+- ./scripts/ci/print-logs.sh

package.json

@@ -46,6 +46,7 @@
     "q": "^1.0.1",
     "protractor": "1.5.x",
     "run-sequence": "^0.3.6",
-    "through2": "^0.6.1"
+    "through2": "^0.6.1",
+    "minimist": "1.1.x"
   }
 }

perf-cloud-secret.template.js

@@ -1,11 +1,7 @@
 module.exports = {
-  auth: {
+  "private_key_id": "1234",
-    "private_key_id": "1234",
+  "private_key": "-----BEGIN PRIVATE KEY-----SOME PRIVATE KEY-----END PRIVATE KEY-----\n",
-    "private_key": "-----BEGIN PRIVATE KEY-----SOME PRIVATE KEY-----END PRIVATE KEY-----\n",
+  "client_email": "SOME_EMAIL@developer.gserviceaccount.com",
-    "client_email": "SOME_EMAIL@developer.gserviceaccount.com",
+  "client_id": "SOME_ID",
-    "client_id": "SOME_ID",
+  "type": "service_account"
-    "type": "service_account"
-  },
-  projectId: 'angular-perf',
-  datasetId: 'benchmarks'
 };

protractor-e2e-shared.js

@@ -1,46 +1,2 @@
-// load traceur runtime as our tests are written in es6
+var config = exports.config = require('./protractor-shared.js').config;
-require('traceur/bin/traceur-runtime.js');
+config.specs = ['dist/cjs/**/*_spec.js'];
-
-var config = exports.config = {
-
-  specs: ['dist/cjs/**/*_spec.js'],
-
-  // Disable waiting for Angular as we don't have an integration layer yet...
-  // TODO(tbosch): Implement a proper debugging API for Ng2.0, remove this here
-  // and the sleeps in all tests.
-  onPrepare: function() {
-    browser.ignoreSynchronization = true;
-    var _get = browser.get;
-    var sleepInterval = process.env.TRAVIS ? 5000 : 2000;
-    browser.get = function() {
-      var result = _get.apply(this, arguments);
-      browser.sleep(sleepInterval);
-      return result;
-    }
-  },
-
-  jasmineNodeOpts: {
-    showColors: true,
-    defaultTimeoutInterval: 30000
-  }
-};
-
-if (process.env.TRAVIS) {
-  config.capabilities = {
-    name: 'Dartium',
-    browserName: 'chrome',
-    chromeOptions: {
-      'binary': process.env.DARTIUM
-    },
-    loggingPrefs: {
-      performance: 'ALL'
-    }
-  };
-} else {
-  config.capabilities = {
-    browserName: 'chrome',
-    loggingPrefs: {
-      performance: 'ALL'
-    }
-  };
-}

protractor-perf-dart2js.conf.js

@@ -1,6 +1,6 @@
 var config = exports.config = require('./protractor-perf-shared.js').config;
-config.params.lang = 'dart';
 config.baseUrl = 'http://localhost:8002/';
+config.params.lang = 'dart';
 // TODO: remove this line when largetable dart has been added
 config.exclude = config.exclude || [];
 config.exclude.push('dist/cjs/e2e_test/benchmarks_external/e2e_test/largetable_perf.js');

protractor-perf-js.conf.js

@@ -1,4 +1,3 @@
 var config = exports.config = require('./protractor-perf-shared.js').config;
-config.params.lang = 'js';
 config.baseUrl = 'http://localhost:8001/';
-
+config.params.lang = 'js';

protractor-perf-shared.js

@@ -1,74 +1,33 @@
+var config = exports.config = require('./protractor-shared.js').config;
 // load traceur runtime as our tests are written in es6
 require('traceur/bin/traceur-runtime.js');
 
 var cloudReporterConfig;
-try {
+if (process.env.CLOUD_SECRET_PATH) {
-  cloudReporterConfig = require('./perf-cloud-secret.js');
+  console.log('using cloud reporter!');
-} catch (e) {
+  cloudReporterConfig = {
-  cloudReporterConfig = null;
+    auth: require(process.env.CLOUD_SECRET_PATH),
+    projectId: 'angular-perf',
+    datasetId: 'benchmarks'
+  };
 }
 
-var config = exports.config = {
+config.specs = ['dist/cjs/**/*_perf.js'];
-
+config.params = {
-  specs: ['dist/cjs/**/*_perf.js'],
+  benchmark: {
-
+    // size of the sample to take
-  params: {
+    sampleSize: 20,
-    benchmark: {
+    timeout: 20000,
-      // size of the sample to take
+    metrics: ['script', 'render', 'gcAmount', 'gcAmountInScript', 'gcTime'],
-      sampleSize: 20,
+    // forces a gc after every run
-      timeout: 20000,
+    forceGc: false,
-      metrics: ['script', 'render', 'gcAmount', 'gcAmountInScript', 'gcTime'],
+    reporters: [
-      // forces a gc after every run
+      require('./dist/cjs/tools/benchpress/src/console_reporter.js'),
-      forceGc: false,
+      cloudReporterConfig ? require('./dist/cjs/tools/benchpress/src/cloud_reporter.js') : null,
-      reporters: [
+    ],
-        require('./dist/cjs/tools/benchpress/src/console_reporter.js'),
+    cloudReporter: cloudReporterConfig,
-        cloudReporterConfig ? require('./dist/cjs/tools/benchpress/src/cloud_reporter.js') : null,
+    scaling: [{
-      ],
+      browserName: /chrome/, value: 0.5
-      cloudReporter: cloudReporterConfig
+    }]
-    }
-  },
-
-  // Disable waiting for Angular as we don't have an integration layer yet...
-  // TODO(tbosch): Implement a proper debugging API for Ng2.0, remove this here
-  // and the sleeps in all tests.
-  onPrepare: function() {
-    browser.ignoreSynchronization = true;
-    var _get = browser.get;
-    var sleepInterval = process.env.TRAVIS ? 5000 : 2000;
-    browser.get = function() {
-      var result = _get.apply(this, arguments);
-      browser.sleep(sleepInterval);
-      return result;
-    }
-  },
-
-  jasmineNodeOpts: {
-    showColors: true,
-    defaultTimeoutInterval: 30000
   }
 };
-
-// TODO: add real mobile devices via a cloud provider that supports appium
-if (process.env.TRAVIS) {
-  config.capabilities = {
-    name: 'Dartium',
-    browserName: 'chrome',
-    chromeOptions: {
-      'binary': process.env.DARTIUM,
-      'args': ['--js-flags=--expose-gc']
-    },
-    loggingPrefs: {
-      performance: 'ALL'
-    }
-  };
-} else {
-  config.capabilities = {
-    browserName: 'chrome',
-    chromeOptions: {
-      'args': ['--js-flags=--expose-gc']
-    },
-    loggingPrefs: {
-      performance: 'ALL'
-    }
-  };
-}

protractor-shared.js

@@ -0,0 +1,69 @@
+// load traceur runtime as our tests are written in es6
+require('traceur/bin/traceur-runtime.js');
+var cmdArgs = require('minimist')(process.argv);
+var cmdLineBrowsers = cmdArgs.browsers ? cmdArgs.browsers.split(',') : [];
+
+var config = exports.config = {
+  // Disable waiting for Angular as we don't have an integration layer yet...
+  // TODO(tbosch): Implement a proper debugging API for Ng2.0, remove this here
+  // and the sleeps in all tests.
+  onPrepare: function() {
+    browser.ignoreSynchronization = true;
+    var _get = browser.get;
+    var sleepInterval = process.env.TRAVIS || process.env.JENKINS_URL ? 5000 : 2000;
+    browser.get = function() {
+      var result = _get.apply(this, arguments);
+      browser.sleep(sleepInterval);
+      return result;
+    }
+  },
+
+  jasmineNodeOpts: {
+    showColors: true,
+    defaultTimeoutInterval: 30000
+  }
+};
+
+var POSSIBLE_CAPS = {
+  Dartium: {
+    name: 'Dartium',
+    browserName: 'chrome',
+    chromeOptions: {
+      'binary': process.env.DARTIUM,
+      'args': ['--js-flags=--expose-gc']
+    },
+    loggingPrefs: {
+      performance: 'ALL'
+    }
+  },
+  ChromeDesktop: {
+    browserName: 'chrome',
+    chromeOptions: {
+      'args': ['--js-flags=--expose-gc']
+    },
+    loggingPrefs: {
+      performance: 'ALL'
+    }
+  },
+  ChromeAndroid: {
+    browserName: 'chrome',
+    chromeOptions: {
+      androidPackage: 'com.android.chrome',
+      'args': ['--js-flags=--expose-gc']
+    },
+    loggingPrefs: {
+      performance: 'ALL'
+    }
+  }
+};
+if (cmdLineBrowsers.length) {
+  config.multiCapabilities = cmdLineBrowsers.map(function(browserName) {
+    var caps = POSSIBLE_CAPS[browserName];
+    if (!caps) {
+      throw new Error('Not configured browser name: '+browserName);
+    }
+    return caps;
+  });
+} else {
+  config.multiCapabilities = [POSSIBLE_CAPS.ChromeDesktop];
+}

scripts/ci/build.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+echo =============================================================================
+# go to project dir
+SCRIPT_DIR=$(dirname $0)
+source $SCRIPT_DIR/env_dart.sh
+cd $SCRIPT_DIR/../..
+
+./node_modules/.bin/gulp build
+
+pub install

scripts/ci/init_android.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+if [[ $PERF_BROWSERS =~ .*Android.* || $E2E_BROWSERS =~ .*Android.* ]]
+then
+  adb usb
+  adb wait-for-device devices
+  adb reverse tcp:8001 tcp:8001
+  adb reverse tcp:8002 tcp:8002
+fi

scripts/ci/install_dart.sh

@@ -16,8 +16,8 @@ if [[ "$AVAILABLE_DART_VERSION" == "1.7.0-dev.2.0" ]]; then
 fi
 
 URL_PREFIX=https://storage.googleapis.com/dart-archive/channels/$CHANNEL/release/$SVN_REVISION
-DART_SDK_URL=$URL_PREFIX/sdk/dartsdk-linux-x64-release.zip
+DART_SDK_URL="$URL_PREFIX/sdk/dartsdk-$ARCH-release.zip"
-DARTIUM_URL=$URL_PREFIX/dartium/dartium-linux-x64-release.zip
+DARTIUM_URL="$URL_PREFIX/dartium/dartium-$ARCH-release.zip"
 
 download_and_unzip() {
   ZIPFILE=${1/*\//}

scripts/ci/test_e2e.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+echo =============================================================================
+# go to project dir
+SCRIPT_DIR=$(dirname $0)
+source $SCRIPT_DIR/env_dart.sh
+cd $SCRIPT_DIR/../..
+
+./node_modules/.bin/webdriver-manager update
+
+function killServer () {
+  kill $serverPid
+}
+
+./node_modules/.bin/gulp serve.js.prod serve.js.dart2js&
+serverPid=$!
+
+trap killServer EXIT
+
+if [[ $PERF_BROWSERS =~ .*Android.* ]]
+then
+  adb usb
+  adb wait-for-device
+  adb reverse tcp:8001 tcp:8001
+  adb reverse tcp:8002 tcp:8002
+fi
+
+./node_modules/.bin/protractor protractor-e2e-js.conf.js --browsers=$E2E_BROWSERS
+./node_modules/.bin/protractor protractor-e2e-dart2js.conf.js --browsers=$E2E_BROWSERS

scripts/ci/test_perf.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+echo =============================================================================
+# go to project dir
+SCRIPT_DIR=$(dirname $0)
+source $SCRIPT_DIR/env_dart.sh
+cd $SCRIPT_DIR/../..
+
+./node_modules/.bin/webdriver-manager update
+
+function killServer () {
+  kill $serverPid
+}
+
+./node_modules/.bin/gulp serve.js.prod serve.js.dart2js&
+serverPid=$!
+
+trap killServer EXIT
+
+./node_modules/.bin/protractor protractor-perf-js.conf.js --browsers=$PERF_BROWSERS
+./node_modules/.bin/protractor protractor-perf-dart2js.conf.js --browsers=$PERF_BROWSERS

scripts/ci/test_unit.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+echo =============================================================================
+# go to project dir
+SCRIPT_DIR=$(dirname $0)
+source $SCRIPT_DIR/env_dart.sh
+cd $SCRIPT_DIR/../..
+
+./node_modules/karma/bin/karma start karma-js.conf \
+        --reporters=dots \
+        --browsers=$KARMA_BROWSERS --single-run
+./node_modules/karma/bin/karma start karma-dart.conf \
+        --reporters=dots \
+        --browsers=$KARMA_BROWSERS --single-run

scripts/jenkins/jenkins_perf.sh

@@ -0,0 +1,20 @@
+set -e
+
+export NVM_DIR="/Users/Shared/jenkins/nvm"
+. "$NVM_DIR/nvm.sh"  # This loads nvm
+export ANDROID_SDK="/Users/Shared/jenkins/android-sdk"
+export PATH+=":$ANDROID_SDK/tools:$ANDROID_SDK/platform-tools"
+export PATH+=":/usr/local/git/bin"
+
+export CHANNEL=stable
+export ARCH=macos-ia32
+export PERF_BROWSERS=ChromeAndroid
+export CLOUD_SECRET_PATH="/Users/Shared/jenkins/keys/perf-cloud-secret"
+
+nvm use 0.10
+
+./scripts/ci/init_android.sh
+./scripts/ci/install_dart.sh
+npm install
+./scripts/ci/build.sh
+./scripts/ci/test_perf.sh

scripts/travis/build.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-set -e
-
-echo =============================================================================
-# go to project dir
-SCRIPT_DIR=$(dirname $0)
-cd $SCRIPT_DIR/../..
-source ./scripts/env.sh
-
-./node_modules/.bin/gulp build
-
-pub install
-
-./node_modules/karma/bin/karma start karma-js.conf \
-        --reporters=dots \
-        --browsers=$BROWSERS --single-run
-./node_modules/karma/bin/karma start karma-dart.conf \
-        --reporters=dots \
-        --browsers=$BROWSERS --single-run
-
-./node_modules/.bin/webdriver-manager update
-
-function killServer () {
-  kill $serverPid
-}
-
-./node_modules/.bin/gulp serve.js.prod serve.js.dart2js&
-serverPid=$!
-
-trap killServer EXIT
-
-./node_modules/.bin/protractor protractor-e2e-js.conf.js
-./node_modules/.bin/protractor protractor-e2e-dart2js.conf.js