refactor(benchmarks): add cloud reporter, add params

- adds console and cloud reporter (via Google BigQuery).
- makes parameters of tests explicit and modifiable.
- removes `detect` and `ignoreGc` mode from benchpress
  as these can result in unstable numbers.

.gitignore

@@ -19,5 +19,7 @@ pubspec.lock
 .idea/
 *.swo
 
+# Don't check in secret files
+*secret.js
 
 /docs/bower_components/

gulpfile.js

@@ -54,6 +54,10 @@ var _HTLM_DEFAULT_SCRIPTS_JS = [
   }
 ];
 
+var _HTML_DEFAULT_SCRIPTS_DART = [
+  {src: '$MODULENAME_WITHOUT_PATH$.dart', mimeType: 'application/dart'},
+  {src: 'packages/browser/dart.js', mimeType: 'text/javascript'}
+];
 
 var CONFIG = {
   dest: {
@@ -72,7 +76,8 @@ var CONFIG = {
   },
   srcFolderMapping: {
     'default': 'lib',
-    '**/benchmark*/**': 'web',
+    '**/benchmarks/**': 'web',
+    '**/benchmarks_external/**': 'web',
     '**/example*/**': 'web'
   },
   deps: {
@@ -83,8 +88,12 @@ var CONFIG = {
       "node_modules/systemjs/lib/extension-register.js",
       "node_modules/zone.js/zone.js",
       "node_modules/zone.js/long-stack-trace-zone.js",
-      "tools/build/runtime_paths.js",
+      "tools/build/snippets/runtime_paths.js",
+      "tools/build/snippets/url_params_to_form.js",
       "node_modules/angular/angular.js"
+    ],
+    dart: [
+      "tools/build/snippets/url_params_to_form.js"
     ]
   },
   transpile: {
@@ -133,14 +142,22 @@ var CONFIG = {
     scriptsPerFolder: {
       js: {
         default: _HTLM_DEFAULT_SCRIPTS_JS,
+        'benchmarks/**':
+          [
+            { src: '/deps/url_params_to_form.js', mimeType: 'text/javascript' }
+          ].concat(_HTLM_DEFAULT_SCRIPTS_JS),
         'benchmarks_external/**':
-          [{ src: '/deps/angular.js', mimeType: 'text/javascript' }].concat(_HTLM_DEFAULT_SCRIPTS_JS)
+          [
+            { src: '/deps/angular.js', mimeType: 'text/javascript' },
+            { src: '/deps/url_params_to_form.js', mimeType: 'text/javascript' }
+          ].concat(_HTLM_DEFAULT_SCRIPTS_JS)
       },
       dart: {
-        default: [
+        default: _HTML_DEFAULT_SCRIPTS_DART,
-          {src: '$MODULENAME_WITHOUT_PATH$.dart', mimeType: 'application/dart'},
+        'benchmarks*/**':
-          {src: 'packages/browser/dart.js', mimeType: 'text/javascript'}
+          [
-        ]
+            { src: '/deps/url_params_to_form.js', mimeType: 'text/javascript' }
+          ].concat(_HTML_DEFAULT_SCRIPTS_DART)
       }
     }
   },
@@ -178,6 +195,11 @@ gulp.task('build/deps.js.prod', deps(gulp, gulpPlugins, {
   dest: CONFIG.dest.js.prod
 }));
 
+gulp.task('build/deps.js.dart2js', deps(gulp, gulpPlugins, {
+  src: CONFIG.deps.dart,
+  dest: CONFIG.dest.js.dart2js
+}));
+
 // ------------
 // transpile
 
@@ -373,7 +395,7 @@ gulp.task('docs/serve', function() {
 // orchestrated targets
 gulp.task('build.dart', function() {
   return runSequence(
-    ['build/transpile.dart', 'build/html.dart'],
+    ['build/deps.js.dart2js', 'build/transpile.dart', 'build/html.dart'],
     'build/pubspec.dart',
     'build/pubbuild.dart',
     'build/analyze.dart'

modules/benchmarks/e2e_test/change_detection_perf.es6

@@ -1,39 +1,31 @@
-"use strict";
+var perfUtil = require('../../e2e_test_lib/e2e_test/perf_util');
-var benchpress = require('../../../tools/benchpress/index.js');
 
 describe('ng2 change detection benchmark', function () {
 
   var URL = 'benchmarks/web/change_detection/change_detection_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(perfUtil.verifyNoBrowserErrors);
 
   it('should log ng stats', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#ng2DetectChanges'],
-      logId: 'ng2.changeDetection'
+      id: 'ng2.changeDetection',
+      params: [{
+        name: 'iterations', value: 500000
+      }]
     });
   });
 
   it('should log baseline stats', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#baselineDetectChanges'],
-      logId: 'baseline.changeDetection'
+      id: 'baseline.changeDetection',
+      params: [{
+        name: 'iterations', value: 500000
+      }]
     });
   });
 
 });
-
-function runClickBenchmark(config) {
-  var buttons = config.buttons.map(function(selector) {
-    return $(selector);
-  });
-  var params = Object.create(browser.params.benchmark);
-  params.logId = browser.params.lang+'.'+config.logId;
-  benchpress.runBenchmark(params, function() {
-    buttons.forEach(function(button) {
-      button.click();
-    });
-  });
-}

modules/benchmarks/e2e_test/change_detection_spec.es6

@@ -1,20 +1,14 @@
-var benchpress = require('../../../tools/benchpress/index.js');
+var testUtil = require('../../e2e_test_lib/e2e_test/test_util');
 
 describe('ng2 change detection benchmark', function () {
 
   var URL = 'benchmarks/web/change_detection/change_detection_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(testUtil.verifyNoBrowserErrors);
 
   it('should not throw errors', function() {
     browser.get(URL);
-    clickAll(['#ng2DetectChanges', '#baselineDetectChanges']);
+    testUtil.clickAll(['#ng2DetectChanges', '#baselineDetectChanges']);
   });
 
 });
-
-function clickAll(buttonSelectors) {
-  buttonSelectors.forEach(function(selector) {
-    $(selector).click();
-  });
-}

modules/benchmarks/e2e_test/compiler_perf.es6

@@ -1,39 +1,31 @@
-"use strict";
+var perfUtil = require('../../e2e_test_lib/e2e_test/perf_util');
-var benchpress = require('../../../tools/benchpress/index.js');
 
 describe('ng2 compiler benchmark', function () {
 
   var URL = 'benchmarks/web/compiler/compiler_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(perfUtil.verifyNoBrowserErrors);
 
   it('should log withBindings stats', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#compileWithBindings'],
-      logId: 'ng2.compile.withBindings'
+      id: 'ng2.compile.withBindings',
+      params: [{
+        name: 'elementCount', selector: '#elementCount', value: 150
+      }]
     });
   });
 
   it('should log noBindings stats', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#compileNoBindings'],
-      logId: 'ng2.compile.noBindings'
+      id: 'ng2.compile.noBindings',
+      params: [{
+        name: 'elementCount', value: 150
+      }]
     });
   });
 
 });
-
-function runClickBenchmark(config) {
-  var buttons = config.buttons.map(function(selector) {
-    return $(selector);
-  });
-  var params = Object.create(browser.params.benchmark);
-  params.logId = browser.params.lang+'.'+config.logId;
-  benchpress.runBenchmark(params, function() {
-    buttons.forEach(function(button) {
-      button.click();
-    });
-  });
-}

modules/benchmarks/e2e_test/compiler_spec.es6

@@ -1,21 +1,14 @@
-"use strict";
+var testUtil = require('../../e2e_test_lib/e2e_test/test_util');
-var benchpress = require('../../../tools/benchpress/index.js');
 
 describe('ng2 compiler benchmark', function () {
 
   var URL = 'benchmarks/web/compiler/compiler_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(testUtil.verifyNoBrowserErrors);
 
   it('should not throw errors', function() {
     browser.get(URL);
-    clickAll(['#compileWithBindings', '#compileNoBindings']);
+    testUtil.clickAll(['#compileWithBindings', '#compileNoBindings']);
   });
 
 });
-
-function clickAll(buttonSelectors) {
-  buttonSelectors.forEach(function(selector) {
-    $(selector).click();
-  });
-}

modules/benchmarks/e2e_test/di_perf.es6

@@ -1,55 +1,53 @@
-"use strict";
+var perfUtil = require('../../e2e_test_lib/e2e_test/perf_util');
-var benchpress = require('../../../tools/benchpress/index.js');
 
 describe('ng2 di benchmark', function () {
 
   var URL = 'benchmarks/web/di/di_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(perfUtil.verifyNoBrowserErrors);
 
   it('should log the stats for getByToken', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#getByToken'],
-      logId: 'ng2.di.getByToken'
+      id: 'ng2.di.getByToken',
+      params: [{
+        name: 'iterations', value: 20000
+      }]
     });
   });
 
   it('should log the stats for getByKey', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#getByKey'],
-      logId: 'ng2.di.getByKey'
+      id: 'ng2.di.getByKey',
+      params: [{
+        name: 'iterations', value: 20000
+      }]
     });
   });
 
   it('should log the stats for getChild', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#getChild'],
-      logId: 'ng2.di.getChild'
+      id: 'ng2.di.getChild',
+      params: [{
+        name: 'iterations', value: 20000
+      }]
     });
   });
 
   it('should log the stats for instantiate', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#instantiate'],
-      logId: 'ng2.di.instantiate'
+      id: 'ng2.di.instantiate',
+      params: [{
+        name: 'iterations', value: 10000
+      }]
     });
   });
 
 });
-
-function runClickBenchmark(config) {
-  var buttons = config.buttons.map(function(selector) {
-    return $(selector);
-  });
-  var params = Object.create(browser.params.benchmark);
-  params.logId = browser.params.lang+'.'+config.logId;
-  benchpress.runBenchmark(params, function() {
-    buttons.forEach(function(button) {
-      button.click();
-    });
-  });
-}

modules/benchmarks/e2e_test/di_spec.es6

@@ -1,20 +1,14 @@
-var benchpress = require('../../../tools/benchpress/index.js');
+var testUtil = require('../../e2e_test_lib/e2e_test/test_util');
 
 describe('ng2 di benchmark', function () {
 
   var URL = 'benchmarks/web/di/di_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(testUtil.verifyNoBrowserErrors);
 
   it('should not throw errors', function() {
     browser.get(URL);
-    clickAll(['#getByToken', '#getByKey', '#getChild', '#instantiate']);
+    testUtil.clickAll(['#getByToken', '#getByKey', '#getChild', '#instantiate']);
   });
 
 });
-
-function clickAll(buttonSelectors) {
-  buttonSelectors.forEach(function(selector) {
-    $(selector).click();
-  });
-}

modules/benchmarks/e2e_test/element_injector_perf.es6

@@ -1,39 +1,31 @@
-"use strict";
+var perfUtil = require('../../e2e_test_lib/e2e_test/perf_util');
-var benchpress = require('../../../tools/benchpress/index.js');
 
 describe('ng2 element injector benchmark', function () {
 
   var URL = 'benchmarks/web/element_injector/element_injector_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(perfUtil.verifyNoBrowserErrors);
 
   it('should log the stats for instantiate', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#instantiate'],
-      logId: 'ng2.elementInjector.instantiate'
+      id: 'ng2.elementInjector.instantiate',
+      params: [{
+        name: 'iterations', value: 20000
+      }]
     });
   });
 
   it('should log the stats for instantiateDirectives', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#instantiateDirectives'],
-      logId: 'ng2.elementInjector.instantiateDirectives'
+      id: 'ng2.elementInjector.instantiateDirectives',
+      params: [{
+        name: 'iterations', value: 20000
+      }]
     });
   });
 
 });
-
-function runClickBenchmark(config) {
-  var buttons = config.buttons.map(function(selector) {
-    return $(selector);
-  });
-  var params = Object.create(browser.params.benchmark);
-  params.logId = browser.params.lang+'.'+config.logId;
-  benchpress.runBenchmark(params, function() {
-    buttons.forEach(function(button) {
-      button.click();
-    });
-  });
-}

modules/benchmarks/e2e_test/element_injector_spec.es6

@@ -1,20 +1,14 @@
-var benchpress = require('../../../tools/benchpress/index.js');
+var testUtil = require('../../e2e_test_lib/e2e_test/test_util');
 
 describe('ng2 element injector benchmark', function () {
 
   var URL = 'benchmarks/web/element_injector/element_injector_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(testUtil.verifyNoBrowserErrors);
 
   it('should not throw errors', function() {
     browser.get(URL);
-    clickAll(['#instantiate', '#instantiateDirectives']);
+    testUtil.clickAll(['#instantiate', '#instantiateDirectives']);
   });
 
 });
-
-function clickAll(buttonSelectors) {
-  buttonSelectors.forEach(function(selector) {
-    $(selector).click();
-  });
-}

modules/benchmarks/e2e_test/selector_perf.es6

@@ -0,0 +1,41 @@
+var perfUtil = require('../../e2e_test_lib/e2e_test/perf_util');
+
+describe('ng2 selector benchmark', function () {
+
+  var URL = 'benchmarks/web/compiler/selector_benchmark.html';
+
+  afterEach(perfUtil.verifyNoBrowserErrors);
+
+  it('should log parse stats', function() {
+    perfUtil.runClickBenchmark({
+      url: URL,
+      buttons: ['#parse'],
+      id: 'ng2.selector.parse',
+      params: [{
+        name: 'selectors', value: 10000
+      }]
+    });
+  });
+
+  it('should log addSelectable stats', function() {
+    perfUtil.runClickBenchmark({
+      buttons: ['#addSelectable'],
+      id: 'ng2.selector.addSelectable',
+      params: [{
+        name: 'selectors', value: 10000
+      }]
+    });
+  });
+
+  it('should log match stats', function() {
+    perfUtil.runClickBenchmark({
+      url: URL,
+      buttons: ['#match'],
+      id: 'ng2.selector.match',
+      params: [{
+        name: 'selectors', value: 10000
+      }]
+    });
+  });
+
+});

modules/benchmarks/e2e_test/selector_spec.es6

@@ -0,0 +1,14 @@
+var testUtil = require('../../e2e_test_lib/e2e_test/test_util');
+
+describe('ng2 selector benchmark', function () {
+
+  var URL = 'benchmarks/web/compiler/selector_benchmark.html';
+
+  afterEach(testUtil.verifyNoBrowserErrors);
+
+  it('should not throw errors', function() {
+    browser.get(URL);
+    testUtil.clickAll(['#parse', '#addSelectable', '#match']);
+  });
+
+});

modules/benchmarks/e2e_test/tree_perf.es6

@@ -1,39 +1,31 @@
-"use strict";
+var perfUtil = require('../../e2e_test_lib/e2e_test/perf_util');
-var benchpress = require('../../../tools/benchpress/index.js');
 
 describe('ng2 tree benchmark', function () {
 
   var URL = 'benchmarks/web/tree/tree_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(perfUtil.verifyNoBrowserErrors);
 
   it('should log the ng stats', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#ng2DestroyDom', '#ng2CreateDom'],
-      logId: 'ng2.tree'
+      id: 'ng2.tree',
+      params: [{
+        name: 'depth', value: 9
+      }]
     });
   });
 
   it('should log the baseline stats', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#baselineDestroyDom', '#baselineCreateDom'],
-      logId: 'baseline.tree'
+      id: 'baseline.tree',
+      params: [{
+        name: 'depth', value: 9
+      }]
     });
   });
 
 });
-
-function runClickBenchmark(config) {
-  var buttons = config.buttons.map(function(selector) {
-    return $(selector);
-  });
-  var params = Object.create(browser.params.benchmark);
-  params.logId = browser.params.lang+'.'+config.logId;
-  benchpress.runBenchmark(params, function() {
-    buttons.forEach(function(button) {
-      button.click();
-    });
-  });
-}

modules/benchmarks/e2e_test/tree_spec.es6

@@ -1,20 +1,14 @@
-var benchpress = require('../../../tools/benchpress/index.js');
+var testUtil = require('../../e2e_test_lib/e2e_test/test_util');
 
 describe('ng2 tree benchmark', function () {
 
   var URL = 'benchmarks/web/tree/tree_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(testUtil.verifyNoBrowserErrors);
 
   it('should not throw errors', function() {
     browser.get(URL);
-    clickAll(['#ng2CreateDom', '#ng2DestroyDom', '#baselineCreateDom', '#baselineDestroyDom']);
+    testUtil.clickAll(['#ng2CreateDom', '#ng2DestroyDom', '#baselineCreateDom', '#baselineDestroyDom']);
   });
 
 });
-
-function clickAll(buttonSelectors) {
-  buttonSelectors.forEach(function(selector) {
-    $(selector).click();
-  });
-}

modules/benchmarks/pubspec.yaml

@@ -12,6 +12,8 @@ dependencies:
     path: ../core
   change_detection:
     path: ../change_detection
+  e2e_test_lib:
+    path: ../e2e_test_lib
   browser: '>=0.10.0 <0.11.0'
 transformers:
 - $dart2js:

modules/benchmarks/src/change_detection/change_detection_benchmark.html

@@ -2,8 +2,19 @@
 <html>
 <body>
 
+<h2>Params</h2>
+<form>
+  Iterations:
+  <input type="number" name="iterations" placeholder="iterations" value="500000">
+  <br>
+  <button>Apply</button>
+</form>
+
+<h2>Actions</h2>
+<p>
 <button id="ng2DetectChanges">Ng2 detect changes</button>
-<button id="baselineDetectChanges">baseline detect changes</button>
+<button id="baselineDetectChanges">baselineDetectChanges</button>
+</p>
 
 $SCRIPTS$
 

modules/benchmarks/src/change_detection/change_detection_benchmark.js

@@ -1,7 +1,7 @@
 import {ListWrapper, MapWrapper} from 'facade/collection';
 import {reflector} from 'reflection/reflection';
 import {isPresent} from 'facade/lang';
-import {document, DOM} from 'facade/dom';
+import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util';
 
 import {
   Lexer,
@@ -12,8 +12,6 @@ import {
 } from 'change_detection/change_detection';
 
 
-var ITERATIONS = 500000;
-
 class Obj {
   field0;
   field1;
@@ -77,7 +75,7 @@ function setUpReflector() {
   });
 }
 
-function setUpBaseline() {
+function setUpBaseline(iterations) {
   function createRow(i) {
     var obj = new Obj();
     var index = i % 10;
@@ -92,7 +90,7 @@ function setUpBaseline() {
 
   var head = createRow(0);
   var current = head;
-  for (var i = 1; i < ITERATIONS; i++) {
+  for (var i = 1; i < iterations; i++) {
     var newRow = createRow(i);
     current.next = newRow;
     current = newRow;
@@ -100,7 +98,7 @@ function setUpBaseline() {
   return head;
 }
 
-function setUpChangeDetection() {
+function setUpChangeDetection(iterations) {
   var dispatcher = new DummyDispatcher();
   var parser = new Parser(new Lexer());
 
@@ -139,7 +137,7 @@ function setUpChangeDetection() {
     proto(9)
   ];
 
-  for (var i = 0; i < ITERATIONS; ++i) {
+  for (var i = 0; i < iterations; ++i) {
     var obj = new Obj();
     var index = i % 10;
     obj.setField(index, i);
@@ -154,11 +152,13 @@ function setUpChangeDetection() {
 }
 
 export function main () {
-  setUpReflector();
+  var iterations = getIntParameter('iterations');
-  var baselineHead = setUpBaseline();
-  var ng2ChangeDetector = setUpChangeDetection();
 
-  function baselineDetectChanges(_) {
+  setUpReflector();
+  var baselineHead = setUpBaseline(iterations);
+  var ng2ChangeDetector = setUpChangeDetection(iterations);
+
+  function baselineDetectChanges() {
     var current = baselineHead;
     while (isPresent(current)) {
       if (current.getter(current.obj) !== current.previousValue) {
@@ -168,12 +168,12 @@ export function main () {
     }
   }
 
-  function ng2DetectChanges(_) {
+  function ng2DetectChanges() {
     ng2ChangeDetector.detectChanges();
   }
 
-  DOM.on(DOM.querySelector(document, '#ng2DetectChanges'), 'click', ng2DetectChanges);
+  bindAction('#ng2DetectChanges', ng2DetectChanges);
-  DOM.on(DOM.querySelector(document, '#baselineDetectChanges'), 'click', baselineDetectChanges);
+  bindAction('#baselineDetectChanges', baselineDetectChanges);
 }
 
 

modules/benchmarks/src/compiler/compiler_benchmark.html

@@ -2,8 +2,19 @@
 <html>
 <body>
 
-<button id="compileWithBindings">Compile template with bindings</button>
+<h2>Params</h2>
-<button id="compileNoBindings">Compile template without bindings</button>
+<form>
+  Elements:
+  <input type="number" name="elements" placeholder="elements" value="150">
+  <br>
+  <button>Apply</button>
+</form>
+
+<h2>Actions</h2>
+<p>
+  <button id="compileWithBindings">CompileWithBindings</button>
+  <button id="compileNoBindings">CompileNoBindings</button>
+</p>
 
 <template id="templateNoBindings">
   <div class="class0 class1 class2 class3 class4 " nodir0="" attr0="value0" nodir1="" attr1="value1" nodir2="" attr2="value2" nodir3="" attr3="value3" nodir4="" attr4="value4">

modules/benchmarks/src/compiler/compiler_benchmark.js

@@ -13,8 +13,7 @@ import {Decorator} from 'core/annotations/annotations';
 import {TemplateConfig} from 'core/annotations/template_config';
 
 import {reflector} from 'reflection/reflection';
-
+import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util';
-var COUNT = 30;
 
 function setupReflector() {
   reflector.registerType(BenchmarkComponent, {
@@ -75,31 +74,33 @@ function setupReflector() {
 }
 
 export function main() {
+  var count = getIntParameter('elements');
+
   setupReflector();
   var reader = new DirectiveMetadataReader();
   var cache = new CompilerCache();
   var compiler = new Compiler(null, reader, new Parser(new Lexer()), cache);
   var annotatedComponent = reader.read(BenchmarkComponent);
 
-  var templateNoBindings = loadTemplate('templateNoBindings', COUNT);
+  var templateNoBindings = loadTemplate('templateNoBindings', count);
-  var templateWithBindings = loadTemplate('templateWithBindings', COUNT);
+  var templateWithBindings = loadTemplate('templateWithBindings', count);
 
-  function compileNoBindings(_) {
+  function compileNoBindings() {
     // Need to clone every time as the compiler might modify the template!
     var cloned = DOM.clone(templateNoBindings);
     cache.clear();
     compiler.compileAllLoaded(null, annotatedComponent, cloned);
   }
 
-  function compileWithBindings(_) {
+  function compileWithBindings() {
     // Need to clone every time as the compiler might modify the template!
     var cloned = DOM.clone(templateWithBindings);
     cache.clear();
     compiler.compileAllLoaded(null, annotatedComponent, cloned);
   }
 
-  DOM.on(DOM.querySelector(document, '#compileNoBindings'), 'click', compileNoBindings);
+  bindAction('#compileNoBindings', compileNoBindings);
-  DOM.on(DOM.querySelector(document, '#compileWithBindings'), 'click', compileWithBindings);
+  bindAction('#compileWithBindings', compileWithBindings);
 }
 
 function loadTemplate(templateId, repeatCount) {

modules/benchmarks/src/compiler/selector_benchmark.html

@@ -2,9 +2,20 @@
 <html>
 <body>
 
+<h2>Params</h2>
+<form>
+  Selectors:
+  <input type="number" name="selectors" placeholder="selectors" value="10000">
+  <br>
+  <button>Apply</button>
+</form>
+
+<h2>Actions</h2>
+<p>
 <button id="parse">Selector.parse</button>
 <button id="addSelectable">Selector.addSelectable</button>
 <button id="match">Selector.match</button>
+</p>
 
 $SCRIPTS$
 

modules/benchmarks/src/compiler/selector_benchmark.js

@@ -1,46 +1,45 @@
-import {document, DOM} from 'facade/dom';
-
 import {SelectorMatcher} from "core/compiler/selector";
 import {CssSelector} from "core/compiler/selector";
 import {StringWrapper, Math} from 'facade/lang';
 import {ListWrapper} from 'facade/collection';
-
+import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util';
-var COUNT = 1000;
 
 export function main() {
+  var count = getIntParameter('selectors');
+
   var fixedMatcher;
   var fixedSelectorStrings = [];
   var fixedSelectors = [];
-  for (var i=0; i<COUNT; i++) {
+  for (var i=0; i<count; i++) {
     ListWrapper.push(fixedSelectorStrings, randomSelector());
   }
-  for (var i=0; i<COUNT; i++) {
+  for (var i=0; i<count; i++) {
     ListWrapper.push(fixedSelectors, CssSelector.parse(fixedSelectorStrings[i]));
   }
   fixedMatcher = new SelectorMatcher();
-  for (var i=0; i<COUNT; i++) {
+  for (var i=0; i<count; i++) {
     fixedMatcher.addSelectable(fixedSelectors[i], i);
   }
 
-  function parse(_) {
+  function parse() {
     var result = [];
-    for (var i=0; i<COUNT; i++) {
+    for (var i=0; i<count; i++) {
       ListWrapper.push(result, CssSelector.parse(fixedSelectorStrings[i]));
     }
     return result;
   }
 
-  function addSelectable(_) {
+  function addSelectable() {
     var matcher = new SelectorMatcher();
-    for (var i=0; i<COUNT; i++) {
+    for (var i=0; i<count; i++) {
       matcher.addSelectable(fixedSelectors[i], i);
     }
     return matcher;
   }
 
-  function match(_) {
+  function match() {
     var matchCount = 0;
-    for (var i=0; i<COUNT; i++) {
+    for (var i=0; i<count; i++) {
       fixedMatcher.match(fixedSelectors[i], (selected) => {
         matchCount += selected;
       });
@@ -48,9 +47,9 @@ export function main() {
     return matchCount;
   }
 
-  DOM.on(DOM.querySelector(document, '#parse'), 'click', parse);
+  bindAction('#parse', parse);
-  DOM.on(DOM.querySelector(document, '#addSelectable'), 'click', addSelectable);
+  bindAction('#addSelectable', addSelectable);
-  DOM.on(DOM.querySelector(document, '#match'), 'click', match);
+  bindAction('#match', match);
 }
 
 function randomSelector() {

modules/benchmarks/src/di/di_benchmark.html

@@ -2,10 +2,21 @@
 <html>
 <body>
 
-<button id="getByToken">Injector.get (token)</button>
+<h2>Params</h2>
-<button id="getByKey">Injector.get (key)</button>
+<form>
-<button id="getChild">Injector.get (grand x 5 child)</button>
+  Iterations:
-<button id="instantiate">Injector.instantiate</button>
+  <input type="number" name="iterations" placeholder="iterations" value="20000">
+  <br>
+  <button>Apply</button>
+</form>
+
+<h2>Actions</h2>
+<p>
+<button id="getByToken">getByToken</button>
+<button id="getByKey">getByKey</button>
+<button id="getChild">getChild</button>
+<button id="instantiate">instantiate</button>
+</div>
 
 $SCRIPTS$
 </body>

modules/benchmarks/src/di/di_benchmark.js

@@ -1,6 +1,6 @@
 import {Injector, Key} from "di/di";
 import {reflector} from 'reflection/reflection';
-import {document, DOM} from 'facade/dom';
+import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util';
 
 var count = 0;
 
@@ -33,6 +33,8 @@ function setupReflector() {
 }
 
 export function main() {
+  var iterations = getIntParameter('iterations');
+
   setupReflector();
   var bindings = [A, B, C, D, E];
   var injector = new Injector(bindings);
@@ -46,37 +48,37 @@ export function main() {
     createChild([]).
     createChild([]);
 
-  function getByToken (_) {
+  function getByToken () {
-    for (var i = 0; i < 20000; ++i) {
+    for (var i = 0; i < iterations; ++i) {
       injector.get(D);
       injector.get(E);
     }
   }
-  function getByKey(_) {
+  function getByKey() {
-    for (var i = 0; i < 20000; ++i) {
+    for (var i = 0; i < iterations; ++i) {
       injector.get(D_KEY);
       injector.get(E_KEY);
     }
   }
 
-  function getChild (_) {
+  function getChild () {
-    for (var i = 0; i < 20000; ++i) {
+    for (var i = 0; i < iterations; ++i) {
       childInjector.get(D);
       childInjector.get(E);
     }
   }
 
-  function instantiate (_) {
+  function instantiate () {
-    for (var i = 0; i < 5000; ++i) {
+    for (var i = 0; i < iterations; ++i) {
       var child = injector.createChild([E]);
       child.get(E);
     }
   }
 
-  DOM.on(DOM.querySelector(document, '#getByToken'), 'click', getByToken);
+  bindAction('#getByToken', getByToken);
-  DOM.on(DOM.querySelector(document, '#getByKey'), 'click', getByKey);
+  bindAction('#getByKey', getByKey);
-  DOM.on(DOM.querySelector(document, '#getChild'), 'click', getChild);
+  bindAction('#getChild', getChild);
-  DOM.on(DOM.querySelector(document, '#instantiate'), 'click', instantiate);
+  bindAction('#instantiate', instantiate);
 }
 
 

modules/benchmarks/src/element_injector/element_injector_benchmark.html

@@ -2,8 +2,19 @@
 <html>
 <body>
 
-<button id="instantiate">ElementInjector.instantiate</button>
+<h2>Params</h2>
-<button id="instantiateDirectives">ElementInjector.instantiateDirectives</button>
+<form>
+  Iterations:
+  <input type="number" name="iterations" placeholder="iterations" value="20000">
+  <br>
+  <button>Apply</button>
+</form>
+
+<h2>Actions</h2>
+<p>
+<button id="instantiate">instantiate</button>
+<button id="instantiateDirectives">instantiateDirectives</button>
+</p>
 
 $SCRIPTS$
 </body>

modules/benchmarks/src/element_injector/element_injector_benchmark.js

@@ -1,10 +1,9 @@
 import {reflector} from 'reflection/reflection';
 import {Injector} from 'di/di';
 import {ProtoElementInjector} from 'core/compiler/element_injector';
-import {document, DOM} from 'facade/dom';
+import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util';
 
 var count = 0;
-var ITERATIONS = 20000;
 
 function setupReflector() {
   reflector.registerType(A, {
@@ -25,6 +24,8 @@ function setupReflector() {
 }
 
 export function main() {
+  var iterations = getIntParameter('iterations');
+
   setupReflector();
   var appInjector = new Injector([]);
 
@@ -32,22 +33,22 @@ export function main() {
   var proto = new ProtoElementInjector(null, 0, bindings);
   var elementInjector = proto.instantiate(null,null);
 
-  function instantiate (_) {
+  function instantiate () {
-    for (var i = 0; i < ITERATIONS; ++i) {
+    for (var i = 0; i < iterations; ++i) {
       var ei = proto.instantiate(null, null);
       ei.instantiateDirectives(appInjector, null, null);
     }
   }
 
-  function instantiateDirectives (_) {
+  function instantiateDirectives () {
-    for (var i = 0; i < ITERATIONS; ++i) {
+    for (var i = 0; i < iterations; ++i) {
       elementInjector.clearDirectives();
       elementInjector.instantiateDirectives(appInjector, null, null);
     }
   }
 
-  DOM.on(DOM.querySelector(document, '#instantiate'), 'click', instantiate);
+  bindAction('#instantiate', instantiate);
-  DOM.on(DOM.querySelector(document, '#instantiateDirectives'), 'click', instantiateDirectives);
+  bindAction('#instantiateDirectives', instantiateDirectives);
 }
 
 class A {

modules/benchmarks/src/tree/tree_benchmark.html

@@ -2,6 +2,14 @@
 <html>
 <body>
 
+<h2>Params</h2>
+<form>
+  Depth:
+  <input type="number" name="depth" placeholder="depth" value="9">
+  <br>
+  <button>Apply</button>
+</form>
+
 <h2>Angular2 tree benchmark</h2>
 <p>
 <button id="ng2DestroyDom">destroyDom</button>

modules/benchmarks/src/tree/tree_benchmark.js

@@ -10,8 +10,7 @@ import {LifeCycle} from 'core/life_cycle/life_cycle';
 import {reflector} from 'reflection/reflection';
 import {DOM, document, window, Element, gc} from 'facade/dom';
 import {isPresent} from 'facade/lang';
-
+import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util';
-var MAX_DEPTH = 9;
 
 function setupReflector() {
   // TODO: Put the general calls to reflector.register... in a shared file
@@ -121,6 +120,8 @@ function setupReflector() {
 }
 
 export function main() {
+ var maxDepth = getIntParameter('depth');
+
  setupReflector();
 
   var app;
@@ -128,7 +129,7 @@ export function main() {
   var baselineRootTreeComponent;
   var count = 0;
 
-  function ng2DestroyDom(_) {
+  function ng2DestroyDom() {
     // TODO: We need an initial value as otherwise the getter for data.value will fail
     // --> this should be already caught in change detection!
     app.initData = new TreeNode('', null, null);
@@ -136,16 +137,16 @@ export function main() {
   }
 
   function profile(create, destroy, name) {
-    return function(_) {
+    return function() {
       window.console.profile(name + ' w GC');
       var duration = 0;
       var count = 0;
       while(count++ < 150) {
         gc();
         var start = window.performance.now();
-        create(_);
+        create();
         duration += window.performance.now() - start;
-        destroy(_);
+        destroy();
       }
       window.console.profileEnd(name + ' w GC');
       window.console.log(`Iterations: ${count}; time: ${duration / count} ms / iteration`);
@@ -155,21 +156,21 @@ export function main() {
       count = 0;
       while(count++ < 150) {
         var start = window.performance.now();
-        create(_);
+        create();
         duration += window.performance.now() - start;
-        destroy(_);
+        destroy();
       }
       window.console.profileEnd(name + ' w/o GC');
       window.console.log(`Iterations: ${count}; time: ${duration / count} ms / iteration`);
     };
   }
 
-  function ng2CreateDom(_) {
+  function ng2CreateDom() {
     var values = count++ % 2 == 0 ?
       ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '*'] :
       ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', '-'];
 
-    app.initData = buildTree(MAX_DEPTH, values, 0);
+    app.initData = buildTree(maxDepth, values, 0);
     changeDetector.detectChanges();
   }
 
@@ -179,33 +180,35 @@ export function main() {
     bootstrap(AppComponent).then((injector) => {
       changeDetector = injector.get(ChangeDetector);
       app = injector.get(AppComponent);
-      DOM.on(DOM.querySelector(document, '#ng2DestroyDom'), 'click', ng2DestroyDom);
+      bindAction('#ng2DestroyDom', ng2DestroyDom);
-      DOM.on(DOM.querySelector(document, '#ng2CreateDom'), 'click', ng2CreateDom);
+      bindAction('#ng2CreateDom', ng2CreateDom);
-      DOM.on(DOM.querySelector(document, '#ng2UpdateDomProfile'), 'click', profile(ng2CreateDom, noop, 'ng2-update'));
+      bindAction('#ng2UpdateDomProfile', profile(ng2CreateDom, noop, 'ng2-update'));
-      DOM.on(DOM.querySelector(document, '#ng2CreateDomProfile'), 'click', profile(ng2CreateDom, ng2DestroyDom, 'ng2-create'));
+      bindAction('#ng2CreateDomProfile', profile(ng2CreateDom, ng2DestroyDom, 'ng2-create'));
     });
   }
 
-  function baselineDestroyDom(_) {
+  function baselineDestroyDom() {
     baselineRootTreeComponent.update(new TreeNode('', null, null));
   }
 
-  function baselineCreateDom(_) {
+  function baselineCreateDom() {
     var values = count++ % 2 == 0 ?
       ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '*'] :
       ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', '-'];
 
-    baselineRootTreeComponent.update(buildTree(MAX_DEPTH, values, 0));
+    baselineRootTreeComponent.update(buildTree(maxDepth, values, 0));
   }
 
   function initBaseline() {
     var tree = DOM.createElement('tree');
     DOM.appendChild(DOM.querySelector(document, 'baseline'), tree);
     baselineRootTreeComponent = new BaseLineTreeComponent(tree);
-    DOM.on(DOM.querySelector(document, '#baselineDestroyDom'), 'click', baselineDestroyDom);
+
-    DOM.on(DOM.querySelector(document, '#baselineCreateDom'), 'click', baselineCreateDom);
+    bindAction('#baselineDestroyDom', baselineDestroyDom);
-    DOM.on(DOM.querySelector(document, '#baselineUpdateDomProfile'), 'click', profile(baselineCreateDom, noop, 'baseline-update'));
+    bindAction('#baselineCreateDom', baselineCreateDom);
-    DOM.on(DOM.querySelector(document, '#baselineCreateDomProfile'), 'click', profile(baselineCreateDom, baselineDestroyDom, 'baseline-create'));
+
+    bindAction('#baselineUpdateDomProfile', profile(baselineCreateDom, noop, 'baseline-update'));
+    bindAction('#baselineCreateDomProfile', profile(baselineCreateDom, baselineDestroyDom, 'baseline-create'));
   }
 
   initNg2();

modules/benchmarks_external/e2e_test/compiler_perf.es6

@@ -1,39 +1,31 @@
-"use strict";
+var perfUtil = require('../../e2e_test_lib/e2e_test/perf_util');
-var benchpress = require('../../../tools/benchpress/index.js');
 
 describe('ng1.x compiler benchmark', function () {
 
   var URL = 'benchmarks_external/web/compiler/compiler_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(perfUtil.verifyNoBrowserErrors);
 
   it('should log withBinding stats', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#compileWithBindings'],
-      logId: 'ng1.compile.withBindings'
+      id: 'ng1.compile.withBindings',
+      params: [{
+        name: 'elementCount', value: 150
+      }]
     });
   });
 
   it('should log noBindings stats', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#compileNoBindings'],
-      logId: 'ng1.compile.noBindings'
+      id: 'ng1.compile.noBindings',
+      params: [{
+        name: 'elementCount', value: 150
+      }]
     });
   });
 
 });
-
-function runClickBenchmark(config) {
-  var buttons = config.buttons.map(function(selector) {
-    return $(selector);
-  });
-  var params = Object.create(browser.params.benchmark);
-  params.logId = browser.params.lang+'.'+config.logId;
-  benchpress.runBenchmark(params, function() {
-    buttons.forEach(function(button) {
-      button.click();
-    });
-  });
-}

modules/benchmarks_external/e2e_test/compiler_spec.es6

@@ -1,20 +1,14 @@
-var benchpress = require('../../../tools/benchpress/index.js');
+var testUtil = require('../../e2e_test_lib/e2e_test/test_util');
 
 describe('ng1.x compiler benchmark', function () {
 
   var URL = 'benchmarks_external/web/compiler/compiler_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(testUtil.verifyNoBrowserErrors);
 
   it('should not throw errors', function() {
     browser.get(URL);
-    clickAll(['#compileWithBindings', '#compileNoBindings']);
+    testUtil.clickAll(['#compileWithBindings', '#compileNoBindings']);
   });
 
 });
-
-function clickAll(buttonSelectors) {
-  buttonSelectors.forEach(function(selector) {
-    $(selector).click();
-  });
-}

modules/benchmarks_external/e2e_test/tree_perf.es6

@@ -1,31 +1,20 @@
-"use strict";
+var perfUtil = require('../../e2e_test_lib/e2e_test/perf_util');
-var benchpress = require('../../../tools/benchpress/index.js');
 
 describe('ng1.x tree benchmark', function () {
 
   var URL = 'benchmarks_external/web/tree/tree_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(perfUtil.verifyNoBrowserErrors);
 
   it('should log the stats', function() {
-    browser.get(URL);
+    perfUtil.runClickBenchmark({
-    runClickBenchmark({
+      url: URL,
       buttons: ['#destroyDom', '#createDom'],
-      logId: 'ng1.tree'
+      id: 'ng1.tree',
+      params: [{
+        name: 'depth', value: 9
+      }]
     });
   });
 
 });
-
-function runClickBenchmark(config) {
-  var buttons = config.buttons.map(function(selector) {
-    return $(selector);
-  });
-  var params = Object.create(browser.params.benchmark);
-  params.logId = browser.params.lang+'.'+config.logId;
-  benchpress.runBenchmark(params, function() {
-    buttons.forEach(function(button) {
-      button.click();
-    });
-  });
-}

modules/benchmarks_external/e2e_test/tree_spec.es6

@@ -1,20 +1,14 @@
-var benchpress = require('../../../tools/benchpress/index.js');
+var testUtil = require('../../e2e_test_lib/e2e_test/test_util');
 
 describe('ng1.x tree benchmark', function () {
 
   var URL = 'benchmarks_external/web/tree/tree_benchmark.html';
 
-  afterEach(benchpress.verifyNoBrowserErrors);
+  afterEach(testUtil.verifyNoBrowserErrors);
 
   it('should not throw errors', function() {
     browser.get(URL);
-    clickAll(['#createDom', '#destroyDom']);
+    testUtil.clickAll(['#createDom', '#destroyDom']);
   });
 
 });
-
-function clickAll(buttonSelectors) {
-  buttonSelectors.forEach(function(selector) {
-    $(selector).click();
-  });
-}

modules/benchmarks_external/pubspec.yaml

@@ -2,6 +2,8 @@ name: benchmarks_external
 environment:
   sdk: '>=1.4.0'
 dependencies:
+  e2e_test_lib:
+    path: ../e2e_test_lib
   angular: ">=1.0.0 <2.0.0"
   browser: '>=0.10.0 <0.11.0'
 transformers:

modules/benchmarks_external/src/compiler/compiler_benchmark.dart

@@ -4,11 +4,12 @@ library compiler_benchmark_ng10;
 import 'package:angular/angular.dart';
 import 'package:angular/application_factory.dart';
 import 'dart:html';
-
+import 'package:e2e_test_lib/benchmark_util.dart';
-var COUNT = 30;
 
 main() {
 
+  var count = getIntParameter('elements');
+
   var m = new Module()
     ..bind(Dir0)
     ..bind(Dir1)
@@ -16,25 +17,25 @@ main() {
     ..bind(Dir3)
     ..bind(Dir4);
 
-  var templateWithBindings = loadTemplate('templateWithBindings', COUNT);
+  var templateWithBindings = loadTemplate('templateWithBindings', count);
-  var templateNoBindings = loadTemplate('templateWithBindings', COUNT);
+  var templateNoBindings = loadTemplate('templateWithBindings', count);
 
   final injector = applicationFactory().addModule(m).run();
   final compiler = injector.get(Compiler);
   final directiveMap = injector.get(DirectiveMap);
 
-  compileWithBindings(_) {
+  compileWithBindings() {
     final cloned = templateWithBindings.clone(true);
     compiler([cloned], directiveMap);
   }
 
-  compileNoBindings(_) {
+  compileNoBindings() {
     final cloned = templateNoBindings.clone(true);
     compiler([cloned], directiveMap);
   }
 
-  document.querySelector('#compileWithBindings').addEventListener('click', compileWithBindings);
+  bindAction('#compileWithBindings', compileWithBindings);
-  document.querySelector('#compileNoBindings').addEventListener('click', compileNoBindings);
+  bindAction('#compileNoBindings', compileNoBindings);
 
 }
 

modules/benchmarks_external/src/compiler/compiler_benchmark.es6

@@ -1,5 +1,5 @@
 // compiler benchmark in AngularJS 1.x
-var COUNT = 30;
+import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util';
 
 export function main() {
   var ngEl = document.createElement('div');
@@ -74,19 +74,20 @@ angular.module('app', [])
   };
 }])
 .run(['$compile', function($compile) {
-  var templateNoBindings = loadTemplate('templateNoBindings', COUNT);
+  var count = getIntParameter('elements');
-  var templateWithBindings = loadTemplate('templateWithBindings', COUNT);
+  var templateNoBindings = loadTemplate('templateNoBindings', count);
+  var templateWithBindings = loadTemplate('templateWithBindings', count);
 
-  document.querySelector('#compileWithBindings').addEventListener('click', compileWithBindings, false);
+  bindAction('#compileWithBindings', compileWithBindings);
-  document.querySelector('#compileNoBindings').addEventListener('click', compileNoBindings, false);
+  bindAction('#compileNoBindings', compileNoBindings);
 
-  function compileNoBindings(_) {
+  function compileNoBindings() {
     // Need to clone every time as the compiler might modify the template!
     var cloned = templateNoBindings.cloneNode(true);
     $compile(cloned);
   }
 
-  function compileWithBindings(_) {
+  function compileWithBindings() {
     // Need to clone every time as the compiler might modify the template!
     var cloned = templateWithBindings.cloneNode(true);
     $compile(cloned);

modules/benchmarks_external/src/compiler/compiler_benchmark.html

@@ -2,8 +2,19 @@
 <html>
 <body>
 
-<button id="compileWithBindings">Compile template with bindings</button>
+<h2>Params</h2>
-<button id="compileNoBindings">Compile template without bindings</button>
+<form>
+  Elements:
+  <input type="number" name="elements" placeholder="elements" value="150">
+  <br>
+  <button>Apply</button>
+</form>
+
+<h2>Actions</h2>
+<p>
+  <button id="compileWithBindings">CompileWithBindings</button>
+  <button id="compileNoBindings">CompileNoBindings</button>
+</p>
 
 <template id="templateNoBindings">
   <div class="class0 class1 class2 class3 class4 " nodir0="" attr0="value0" nodir1="" attr1="value1" nodir2="" attr2="value2" nodir3="" attr3="value3" nodir4="" attr4="value4">

modules/benchmarks_external/src/tree/tree_benchmark.dart

@@ -4,8 +4,7 @@ library tree_benchmark_ng10;
 import 'package:angular/angular.dart';
 import 'package:angular/application_factory.dart';
 import 'dart:html';
-
+import 'package:e2e_test_lib/benchmark_util.dart';
-var MAX_DEPTH = 9;
 
 setup() {
 
@@ -20,29 +19,31 @@ setup() {
 }
 
 main() {
+  var maxDepth = getIntParameter('depth');
+
   final injector = setup();
   final zone = injector.get(VmTurnZone);
   final rootScope = injector.get(Scope);
   var count = 0;
 
-  destroyDom(_) {
+  destroyDom() {
     zone.run(() {
       rootScope.context['initData'] = new TreeNode('');
     });
   }
 
-  createDom(_) {
+  createDom() {
     zone.run(() {
       var values = count++ % 2 == 0 ?
         ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '*'] :
         ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', '-'];
 
-      rootScope.context['initData'] = buildTree(MAX_DEPTH, values, 0);
+      rootScope.context['initData'] = buildTree(maxDepth, values, 0);
     });
   }
 
-  document.querySelector('#destroyDom').addEventListener('click', destroyDom);
+  bindAction('#destroyDom', destroyDom);
-  document.querySelector('#createDom').addEventListener('click', createDom);
+  bindAction('#createDom', createDom);
 }
 
 @Component(

modules/benchmarks_external/src/tree/tree_benchmark.es6

@@ -1,5 +1,5 @@
 // tree benchmark in AngularJS 1.x
-var MAX_DEPTH = 9;
+import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util';
 
 export function main() {
   angular.bootstrap(document.body, ['app']);
@@ -57,24 +57,24 @@ angular.module('app', [])
 }])
 .run(['$rootScope', function($rootScope) {
   var count = 0;
+  var maxDepth = getIntParameter('depth');
 
-  document.querySelector('#destroyDom').addEventListener('click', destroyDom, false);
+  bindAction('#destroyDom', destroyDom);
-  document.querySelector('#createDom').addEventListener('click', createDom, false);
+  bindAction('#createDom', createDom);
 
-  function destroyDom(_) {
+  function destroyDom() {
     $rootScope.$apply(function() {
       $rootScope.initData = new TreeNode('', null, null);
     });
   }
 
-  function createDom(_) {
+  function createDom() {
-    var maxDepth = MAX_DEPTH;
     var values = count++ % 2 == 0 ?
       ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '*'] :
       ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', '-'];
 
     $rootScope.$apply(function() {
-      $rootScope.initData = buildTree(MAX_DEPTH, values, 0);
+      $rootScope.initData = buildTree(maxDepth, values, 0);
     });
   }
 }]);

modules/benchmarks_external/src/tree/tree_benchmark.html

@@ -2,6 +2,14 @@
 <html>
 <body>
 
+<h2>Params</h2>
+<form>
+  Depth:
+  <input type="number" name="depth" placeholder="depth" value="9">
+  <br>
+  <button>Apply</button>
+</form>
+
 <h2>AngularJS/Dart 1.x tree benchmark</h2>
 <p>
 <button id="destroyDom">destroyDom</button>

modules/e2e_test_lib/e2e_test/perf_util.es6

@@ -0,0 +1,25 @@
+var benchpress = require('../../../tools/benchpress/index.js');
+var webdriver = require('protractor/node_modules/selenium-webdriver');
+
+module.exports = {
+  runClickBenchmark: runClickBenchmark,
+  verifyNoBrowserErrors: benchpress.verifyNoBrowserErrors
+};
+
+function runClickBenchmark(config) {
+  var url = encodeURI(config.url + '?' + config.params.map(function(param) {
+    return param.name + '=' + param.value;
+  }).join('&'));
+  browser.get(url);
+  var buttons = config.buttons.map(function(selector) {
+    return $(selector);
+  });
+  var benchmarkConfig = Object.create(browser.params.benchmark);
+  benchmarkConfig.id = browser.params.lang+'.'+config.id;
+  benchmarkConfig.params = config.params;
+  benchpress.runBenchmark(benchmarkConfig, function() {
+    buttons.forEach(function(button) {
+      button.click();
+    });
+  });
+}

modules/e2e_test_lib/e2e_test/test_util.es6

@@ -0,0 +1,12 @@
+var benchpress = require('../../../tools/benchpress/index.js');
+
+module.exports = {
+  verifyNoBrowserErrors: benchpress.verifyNoBrowserErrors,
+  clickAll: clickAll
+};
+
+function clickAll(buttonSelectors) {
+  buttonSelectors.forEach(function(selector) {
+    $(selector).click();
+  });
+}

modules/e2e_test_lib/pubspec.yaml

@@ -0,0 +1,7 @@
+name: e2e_test_lib
+environment:
+  sdk: '>=1.4.0'
+dependencies:
+  facade:
+    path: ../facade
+  browser: '>=0.10.0 <0.11.0'

modules/e2e_test_lib/src/benchmark_util.js

@@ -0,0 +1,17 @@
+import {DOM, document, location} from 'facade/dom';
+import {NumberWrapper, BaseException, isBlank} from 'facade/lang';
+
+export function getIntParameter(name:string) {
+  var el = DOM.querySelector(document, `input[name="${name}"]`);
+  if (isBlank(el)) {
+    throw new BaseException(`Could not find and input field with name ${name}`);
+  }
+  return NumberWrapper.parseInt(el.value, 10);
+}
+
+export function bindAction(selector:string, callback:Function) {
+  var el = DOM.querySelector(document, selector);
+  DOM.on(el, 'click', function(_) {
+    callback();
+  });
+}

package.json

@@ -12,7 +12,9 @@
     "systemjs": "^0.9.1",
     "traceur": "vojtajina/traceur-compiler#disable-getters-setters",
     "which": "~1",
-    "zone.js": "0.3.0"
+    "zone.js": "0.3.0",
+    "googleapis": "1.0.x",
+    "node-uuid": "1.4.x"
   },
   "devDependencies": {
     "bower": "^1.3.12",

perf-cloud-secret.template.js

@@ -0,0 +1,11 @@
+module.exports = {
+  auth: {
+    "private_key_id": "1234",
+    "private_key": "-----BEGIN PRIVATE KEY-----SOME PRIVATE KEY-----END PRIVATE KEY-----\n",
+    "client_email": "SOME_EMAIL@developer.gserviceaccount.com",
+    "client_id": "SOME_ID",
+    "type": "service_account"
+  },
+  projectId: 'angular-perf',
+  datasetId: 'benchmarks'
+};

protractor-perf-shared.js

@@ -1,6 +1,13 @@
 // load traceur runtime as our tests are written in es6
 require('traceur/bin/traceur-runtime.js');
 
+var cloudReporterConfig;
+try {
+  cloudReporterConfig = require('./perf-cloud-secret.js');
+} catch (e) {
+  cloudReporterConfig = null;
+}
+
 var config = exports.config = {
 
   specs: ['dist/cjs/**/*_perf.js'],
@@ -8,16 +15,16 @@ var config = exports.config = {
   params: {
     benchmark: {
       // size of the sample to take
-      sampleSize: 10,
+      sampleSize: 20,
-      targetCoefficientOfVariation: 4,
       timeout: 20000,
       metrics: ['script', 'render', 'gcAmount', 'gcAmountInScript', 'gcTime'],
-      // run mode of the benchmark:
+      // forces a gc after every run
-      // - detect: auto detect whether to force gc
+      forceGc: false,
-      // - forceGc: forces a gc before every run and ignores no runs
+      reporters: [
-      // - noGcInScript: ignore runs that have gc while a script was executing
+        require('./dist/cjs/tools/benchpress/src/console_reporter.js'),
-      // - plain: does not force nor ignore runs
+        cloudReporterConfig ? require('./dist/cjs/tools/benchpress/src/cloud_reporter.js') : null,
-      mode: 'detect'
+      ],
+      cloudReporter: cloudReporterConfig
     }
   },
 

tools/benchpress/src/benchmark.es6

@@ -1,6 +1,7 @@
-var stats = require('./stats');
+var statistics = require('./statistics');
-var reporter = require('./reporter');
 var commands = require('./commands');
+var nodeUuid = require('node-uuid');
+var webdriver = require('protractor/node_modules/selenium-webdriver');
 
 var SUPPORTED_METRICS = {
   script: true,
@@ -12,52 +13,6 @@ var SUPPORTED_METRICS = {
   render: true
 };
 
-var RUN_MODE = {
-  detect: function(prevState, benchmarkData, iterationIndex) {
-    var gcInScriptCount = prevState.gcInScriptCount || 0;
-    if (benchmarkData.gcAmountInScript) {
-      gcInScriptCount++;
-    }
-    var ignoreRun = !!benchmarkData.gcAmountInScript;
-    var nextMode = RUN_MODE.detect;
-    if (iterationIndex > 10) {
-      if (gcInScriptCount / iterationIndex > 0.7) {
-        nextMode = RUN_MODE.forceGc;
-      } else {
-        nextMode = RUN_MODE.noGcInScript;
-      }
-    }
-    return {
-      forceGc: false,
-      ignoreRun: ignoreRun,
-      gcInScriptCount: gcInScriptCount,
-      nextMode: nextMode
-    };
-  },
-  forceGc: function() {
-    return {
-      forceGc: true,
-      ignoreRun: false,
-      nextMode: RUN_MODE.forceGc
-    }
-  },
-  noGcInScript: function(prevState, benchmarkData) {
-    var ignoreRun = !!benchmarkData.gcAmountInScript;
-    return {
-      forceGc: false,
-      ignoreRun: ignoreRun,
-      nextMode: RUN_MODE.noGcInScript
-    }
-  },
-  plain: function() {
-    return {
-      forceGc: false,
-      ignoreRun: false,
-      nextMode: RUN_MODE.plain
-    }
-  }
-};
-
 var nextTimestampId = 0;
 
 module.exports = {
@@ -66,133 +21,139 @@ module.exports = {
 };
 
 function runBenchmark(config, workCallback) {
-  config.metrics.forEach(function(metric) {
+  var sampleId = nodeUuid.v1();
+  var reporters = config.reporters.map(function(Class) {
+    return new Class(sampleId, config);
+  });
+  var scriptMetricIndex = -1;
+  config.metrics.forEach(function(metric, index) {
     if (!(metric in SUPPORTED_METRICS)) {
       throw new Error('Metric '+metric+' is not suported by benchpress right now');
     }
+    if (metric === 'script') {
+      scriptMetricIndex = index;
+    }
   });
-  var ROW_FORMAT = ['%-40s', '%12s'].concat(config.metrics.map(function() {
+  if (scriptMetricIndex === -1) {
-    return '%12s';
+    throw new Error('Metric "script" needs to be included in the metrics');
-  })).join(' | ');
+  }
-
-  var benchmarkStatsAggregator = stats.createObjectStatsAggregator(config.metrics, config.sampleSize);
 
   var startTime = Date.now();
-  startLoop().then(endLoop);
+  commands.gc();
-
+  reporters.forEach(function(reporter) {
-  function startLoop(gcData) {
+    reporter.begin();
-    reporter.printHeading('SCRIPT DATA: sampling size '+config.sampleSize);
-    reporter.printTableHeader(ROW_FORMAT, ['name', 'action'].concat(config.metrics));
-    if (!(config.mode in RUN_MODE)) {
-      throw new Error('Unknown mode '+config.mode);
-    }
-    return loop(0, {
-      forceGc: false,
-      ignoreRun: false,
-      nextMode: RUN_MODE[config.mode]
   });
-  }
+  return measureLoop({
-
+    index: 0,
-  function endLoop(stats) {
+    prevSample: [],
-    reporter.printTableFooter(ROW_FORMAT, [config.logId, '']
+    endAfterRun: false,
-      .concat(formatObjectStats(stats, config.metrics))
+    work: function() {
-    );
-    return config.metrics.map(function(metric) {
-      return stats[metric];
-    });
-  }
-
-  function loop(iterationIndex, modeState) {
-    return measureTime(function() {
       workCallback();
-      if (modeState.forceGc) {
+      if (this.endAfterRun || config.forceGc) {
-        // For fast tests that don't create a lot of garbage,
-        // we don't want to force gc before every run as that
-        // can slow down the script execution time (even when we subtract
-        // the gc time)!
-        // Note: we need to call gc AFTER the actual test so the
-        // gc amount is added to the current test run!
         commands.gc();
       }
-    }).then(function(benchmarkData) {
+    },
-      modeState = modeState.nextMode(modeState, benchmarkData, iterationIndex);
+    process: function(data) {
-      var action = '';
+      var measuredValues = config.metrics.map(function(metric) {
-      if (modeState.ignoreRun) {
+        return data.stats[metric];
-        action = 'ignore';
+      });
-      } else if (modeState.forceGc) {
+      var reporterData = {
-        action = 'forceGc';
+        values: measuredValues,
+        index: this.index,
+        records: data.records,
+        forceGc: this.endAfterRun || config.forceGc
+      };
+      reporters.forEach(function(reporter) {
+        reporter.add(reporterData);
+      });
+
+      var newSample = this.prevSample.concat([reporterData]);
+      if (newSample.length > config.sampleSize) {
+        newSample = newSample.slice(newSample.length - config.sampleSize);
       }
-      reporter.printRow(ROW_FORMAT, [config.logId + '#' + iterationIndex, action]
+
-        .concat(formatObjectData(benchmarkData, config.metrics))
+      var result = null;
+      var xValues = [];
+      var yValues = [];
+      newSample.forEach(function(data, index) {
+        // For now, we only use the array index as x value.
+        // TODO(tbosch): think about whether we should use time here instead
+        xValues.push(index);
+        yValues.push(data.values[scriptMetricIndex]);
+      });
+      var regressionSlope = statistics.getRegressionSlope(
+        xValues, statistics.calculateMean(xValues),
+        yValues, statistics.calculateMean(yValues)
       );
+      // TODO(tbosch): ask someone who really understands statistics whether this is reasonable
+      // When we detect that we are not getting slower any more,
+      // we do one more round where we force gc so we get all the gc data before we stop.
+      var endAfterNextRun = ((Date.now() - startTime > config.timeout) ||
+          (newSample.length === config.sampleSize && regressionSlope >= 0));
+      return {
+        index: this.index+1,
+        work: this.work,
+        process: this.process,
+        endAfterRun: endAfterNextRun,
+        result: this.endAfterRun ? newSample : null,
+        prevSample: newSample
+      };
+    }
+  }).then(function(stableSample) {
+    reporters.forEach(function(reporter) {
+      reporter.end(stableSample);
+    });
+  });
+}
 
-      var benchmarkStats;
+function measureLoop(startState) {
-      if (modeState.ignoreRun) {
+  var startTimestampId = (nextTimestampId++).toString();
-        benchmarkStats = benchmarkStatsAggregator.current;
+  commands.timelineTimestamp(startTimestampId);
+
+  return next(startTimestampId, startState, []);
+
+  function next(startTimestampId, state, lastRecords) {
+    state.work();
+    var endTimestampId = (nextTimestampId++).toString();
+    commands.timelineTimestamp(endTimestampId);
+
+    return readStats(startTimestampId, endTimestampId, lastRecords).then(function(data) {
+      var nextState = state.process({
+        stats: data.stats,
+        records: data.records
+      });
+      if (nextState.result) {
+        return nextState.result;
       } else {
-        benchmarkStats = benchmarkStatsAggregator(benchmarkData);
+        return next(endTimestampId, nextState, data.lastRecords);
-      }
-
-      if (Date.now() - startTime > config.timeout) {
-        return benchmarkStats;
-      }
-      if (benchmarkStats &&
-        (
-          benchmarkStats.script.count >= config.sampleSize &&
-          benchmarkStats.script.coefficientOfVariation < config.targetCoefficientOfVariation)
-        ) {
-        return benchmarkStats
-      }
-      return loop(iterationIndex+1, modeState);
-    });
-  }
-}
-
-function formatObjectData(data, props) {
-  return props.map(function(prop) {
-    var val = data[prop];
-    if (typeof val === 'number') {
-      return val.toFixed(2);
-    } else {
-      return val;
       }
     });
-}
+  }
 
-function formatObjectStats(stats, props) {
+  function readStats(startTimestampId, endTimestampId, lastRecords) {
-  return props.map(function(prop) {
+    return commands.timelineRecords().then(function(newRecords) {
-    var entry = stats[prop];
+      var records = lastRecords.concat(newRecords);
-    return entry.mean.toFixed(2) + '\u00B1' + entry.coefficientOfVariation.toFixed(0)+ '%';
+      var stats = sumTimelineRecords(records, startTimestampId, endTimestampId);
-  });
+      if (stats.timeStamps.indexOf(startTimestampId) === -1 ||
-}
+          stats.timeStamps.indexOf(endTimestampId) === -1) {
-
-function measureTime(callback) {
-  var startId = (nextTimestampId++).toString();
-  var endId = (nextTimestampId++).toString();
-  commands.timelineTimestamp(startId);
-  callback();
-  commands.timelineTimestamp(endId);
-  var allRecords = [];
-  return readResult();
-
-  function readResult() {
-    return commands.timelineRecords().then(function(records) {
-      allRecords.push.apply(allRecords, records);
-      var stats = sumTimelineRecords(allRecords, startId, endId);
-      if (stats.timeStamps.indexOf(startId) === -1 ||
-          stats.timeStamps.indexOf(endId) === -1) {
         // Sometimes the logs have not yet arrived at the webdriver
-        // server from the browser.
+        // server from the browser, so we need to wait
+        // TODO(tbosch): This seems to be a bug in chrome / chromedriver!
         // And sometimes, just waiting is not enough, so we
         // execute a dummy js function :-(
         browser.executeScript('1+1');
         browser.sleep(100);
-        return readResult();
+        return readStats(startTimestampId, endTimestampId, records);
       } else {
-        return stats;
+        return {
+          stats: stats,
+          records: records,
+          lastRecords: newRecords
+        };
       }
     });
   }
+
 }
 
 function sumTimelineRecords(records, startTimeStampId, endTimeStampId) {

tools/benchpress/src/cloud_reporter.es6

@@ -0,0 +1,224 @@
+var google = require('googleapis');
+var bigquery = google.bigquery('v2');
+var webdriver = require('protractor/node_modules/selenium-webdriver');
+
+var HEADER_FIELDS = [
+  {
+    "name": 'runId',
+    "type": 'STRING',
+    "description": 'uuid for the benchmark run'
+  },
+  {
+    "name": 'index',
+    "type": 'INTEGER',
+    "description": 'index within the sample'
+  },
+  {
+    "name": 'creationTime',
+    "type": 'TIMESTAMP'
+  },
+  {
+    "name": 'browser',
+    "type": 'STRING',
+    "description": 'navigator.platform'
+  },
+  {
+    "name": 'forceGc',
+    "type": 'BOOLEAN',
+    "description": 'whether gc was forced at end of action'
+  }
+];
+
+class CloudReporter {
+  constructor(runId, benchmarkConfig) {
+    this.stableRowsTableConfig = createTableConfig(benchmarkConfig, '_stable');
+    this.allRowsTableConfig = createTableConfig(benchmarkConfig, '_all')
+    this.authConfig = benchmarkConfig.cloudReporter.auth;
+    this.benchmarkConfig = benchmarkConfig;
+    this.runId = runId;
+    this.allRows = [];
+    var self = this;
+    browser.executeScript('return navigator.userAgent').then(function(userAgent) {
+      self.browserUserAgent = userAgent;
+    });
+
+  }
+  begin() {
+    var self = this;
+    var flow = browser.driver.controlFlow();
+    flow.execute(function() {
+      return authenticate(self.authConfig).then(function(authClient) {
+        self.authClient = authClient;
+      });
+    });
+    flow.execute(function() {
+      return webdriver.promise.all([
+        getOrCreateTable(self.authClient, self.allRowsTableConfig),
+        getOrCreateTable(self.authClient, self.stableRowsTableConfig)
+      ]);
+    });
+  }
+  add(data) {
+    this.allRows.push(this._convertToTableRow(data));
+  }
+  end(stableSample) {
+    var self = this;
+    var flow = browser.driver.controlFlow();
+    var stableRows = stableSample.map(function(data) {
+      return self._convertToTableRow(data);
+    });
+    flow.execute(function() {
+      return webdriver.promise.all([
+        insertRows(self.authClient, self.stableRowsTableConfig, stableRows),
+        insertRows(self.authClient, self.allRowsTableConfig, self.allRows)
+      ]);
+    });
+  }
+  _convertToTableRow(benchpressRow) {
+    var tableRow = {
+      runId: this.runId,
+      index: benchpressRow.index,
+      creationTime: new Date(),
+      browser: this.browserUserAgent,
+      forceGc: benchpressRow.forceGc
+    };
+    this.benchmarkConfig.params.forEach(function(param) {
+      tableRow['p_'+param.name] = param.value;
+    });
+    this.benchmarkConfig.metrics.forEach(function(metric, index) {
+      tableRow['m_'+metric] = benchpressRow.values[index];
+    });
+    return tableRow;
+  }
+}
+
+function createTableConfig(benchmarkConfig, tableSuffix) {
+  var tableId = (benchmarkConfig.id+tableSuffix).replace(/\./g, '_');
+  return {
+    projectId: benchmarkConfig.cloudReporter.projectId,
+    datasetId: benchmarkConfig.cloudReporter.datasetId,
+    table: {
+      id: tableId,
+      fields: HEADER_FIELDS
+        .concat(benchmarkConfig.params.map(function(param) {
+          return {
+            "name": 'p_'+param.name,
+            "type": 'FLOAT'
+          };
+        }))
+        .concat(benchmarkConfig.metrics.map(function(metricName) {
+          return {
+            "name": 'm_'+metricName,
+            "type": 'FLOAT'
+          };
+        }))
+    }
+  };
+}
+
+function getOrCreateTable(authClient, tableConfig) {
+  return getTable(authClient, tableConfig).then(null, function(err) {
+    // create the table if it does not exist
+    return createTable(authClient, tableConfig);
+  });
+}
+
+function authenticate(authConfig) {
+  var authClient = new google.auth.JWT(
+    authConfig['client_email'],
+    null,
+    authConfig['private_key'],
+    ['https://www.googleapis.com/auth/bigquery'],
+    // User to impersonate (leave empty if no impersonation needed)
+    null);
+
+  var defer = webdriver.promise.defer();
+  authClient.authorize(makeNodeJsResolver(defer));
+  return defer.promise.then(function() {
+    return authClient;
+  });
+}
+
+function getTable(authClient, tableConfig) {
+  // see https://cloud.google.com/bigquery/docs/reference/v2/tables/get
+  var params = {
+    auth: authClient,
+    projectId: tableConfig.projectId,
+    datasetId: tableConfig.datasetId,
+    tableId: tableConfig.table.id
+  };
+  var defer = webdriver.promise.defer();
+  bigquery.tables.get(params, makeNodeJsResolver(defer));
+  return defer.promise;
+}
+
+function createTable(authClient, tableConfig) {
+  // see https://cloud.google.com/bigquery/docs/reference/v2/tables
+  // see https://cloud.google.com/bigquery/docs/reference/v2/tables#resource
+  var params = {
+    auth: authClient,
+    projectId: tableConfig.projectId,
+    datasetId: tableConfig.datasetId,
+    resource: {
+      "kind": "bigquery#table",
+      "tableReference": {
+        projectId: tableConfig.projectId,
+        datasetId: tableConfig.datasetId,
+        tableId: tableConfig.table.id
+      },
+      "schema": {
+        "fields": tableConfig.table.fields
+      }
+    }
+  };
+  var defer = webdriver.promise.defer();
+  bigquery.tables.insert(params, makeNodeJsResolver(defer));
+  return defer.promise;
+}
+
+function insertRows(authClient, tableConfig, rows) {
+  // see https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll
+  var params = {
+    auth: authClient,
+    projectId: tableConfig.projectId,
+    datasetId: tableConfig.datasetId,
+    tableId: tableConfig.table.id,
+    resource: {
+      "kind": "bigquery#tableDataInsertAllRequest",
+      "rows": rows.map(function(row) {
+        return {
+          json: row
+        }
+      })
+    }
+  };
+  var defer = webdriver.promise.defer();
+  bigquery.tabledata.insertAll(params, makeNodeJsResolver(defer));
+  return defer.promise.then(function(result) {
+    if (result.insertErrors) {
+      throw result.insertErrors.map(function(err) {
+        return err.errors.map(function(err) {
+          return err.message;
+        }).join('\n');
+      }).join('\n');
+    }
+  });
+}
+
+function makeNodeJsResolver(defer) {
+  return function(err, result) {
+    if (err) {
+      // Normalize errors messages from BigCloud so that they show up nicely
+      if (err.errors) {
+        err = err.errors.map(function(err) {
+          return err.message;
+        }).join('\n');
+      }
+      defer.reject(err);
+    } else {
+      defer.fulfill(result);
+    }
+  }
+}
+
+module.exports = CloudReporter;

tools/benchpress/src/console_reporter.es6

@@ -0,0 +1,77 @@
+var vsprintf = require("sprintf-js").vsprintf;
+var statistics = require("./statistics");
+
+var HEADER_SEPARATORS = ['----', '----', '----', '----', '----', '----', '----'];
+var FOOTER_SEPARATORS = ['====', '====', '====', '====', '====', '====', '===='];
+
+class ConsoleReporter {
+  constructor(runId, config) {
+    this.config = config;
+    this.runId = runId;
+    this.rowFormat = ['%12s'].concat(config.metrics.map(function() {
+      return '%12s';
+    })).join(' | ');
+  }
+  begin() {
+    printHeading('BENCHMARK '+this.config.id);
+    console.log('sample size', this.config.sampleSize);
+    console.log('run id', this.runId);
+    console.log('params', JSON.stringify(this.config.params, null, '  '));
+    printTableHeader(this.rowFormat, ['index', 'forceGc'].concat(this.config.metrics));
+  }
+  add(data) {
+    var values = data.values;
+    var index = data.index;
+    printRow(this.rowFormat, ['#' + index, data.forceGc]
+      .concat(formatValues(values))
+    );
+  }
+  end(stableSample) {
+    printTableFooter(this.rowFormat, [this.config.id, '']
+      .concat(formatSample(stableSample, this.config.metrics)));
+  }
+}
+
+function formatValues(values) {
+  return values.map(function(val) {
+    if (typeof val === 'number') {
+      return val.toFixed(2);
+    } else {
+      return val;
+    }
+  });
+}
+
+function formatSample(sample, metrics) {
+  return metrics.map(function(_, metricIndex) {
+    var metricSample = sample.map(function(row) {
+      return row.values[metricIndex];
+    });
+    var mean = statistics.calculateMean(metricSample);
+    var coefficientOfVariation = statistics.calculateCoefficientOfVariation(metricSample, mean);
+    return mean.toFixed(2) + '\u00B1' + coefficientOfVariation.toFixed(0)+ '%';
+  });
+}
+
+function printHeading(title) {
+  console.log('\n');
+  console.log('## '+title);
+}
+
+function printTableHeader(format, values) {
+  printRow(format, values);
+  // TODO(tbosch): generate separators dynamically based on the format!
+  printRow(format, HEADER_SEPARATORS);
+}
+
+function printTableFooter(format, values) {
+  // TODO(tbosch): generate separators dynamically based on the format!
+  printRow(format, FOOTER_SEPARATORS);
+  printRow(format, values);
+}
+
+function printRow(format, values) {
+  console.log(vsprintf(format, values));
+}
+
+module.exports = ConsoleReporter;

tools/benchpress/src/reporter.es6

@@ -1,32 +0,0 @@
-var vsprintf = require("sprintf-js").vsprintf;
-
-var HEADER_SEPARATORS = ['----', '----', '----', '----', '----', '----', '----'];
-var FOOTER_SEPARATORS = ['====', '====', '====', '====', '====', '====', '===='];
-
-module.exports = {
-  printHeading: printHeading,
-  printTableHeader: printTableHeader,
-  printTableFooter: printTableFooter,
-  printRow: printRow
-};
-
-function printHeading(title) {
-  console.log('\n');
-  console.log('## '+title);
-}
-
-function printTableHeader(format, values) {
-  printRow(format, values);
-  // TODO(tbosch): generate separators dynamically based on the format!
-  printRow(format, HEADER_SEPARATORS);
-}
-
-function printTableFooter(format, values) {
-  // TODO(tbosch): generate separators dynamically based on the format!
-  printRow(format, FOOTER_SEPARATORS);
-  printRow(format, values);
-}
-
-function printRow(format, values) {
-  console.log(vsprintf(format, values));
-}

tools/benchpress/src/statistics.es6

@@ -0,0 +1,37 @@
+module.exports = {
+  calculateCoefficientOfVariation: calculateCoefficientOfVariation,
+  calculateMean: calculateMean,
+  calculateStandardDeviation: calculateStandardDeviation,
+  getRegressionSlope: getRegressionSlope
+};
+
+function calculateCoefficientOfVariation(sample, mean) {
+  return calculateStandardDeviation(sample, mean) / mean * 100;
+}
+
+function calculateMean(sample) {
+  var total = 0;
+  sample.forEach(function(x) { total += x; });
+  return total / sample.length;
+}
+
+function calculateStandardDeviation(sample, mean) {
+  var deviation = 0;
+  sample.forEach(function(x) {
+    deviation += Math.pow(x - mean, 2);
+  });
+  deviation = deviation / (sample.length);
+  deviation = Math.sqrt(deviation);
+  return deviation;
+}
+
+function getRegressionSlope(xValues, xMean, yValues, yMean) {
+  // See http://en.wikipedia.org/wiki/Simple_linear_regression
+  var dividendSum = 0;
+  var divisorSum = 0;
+  for (var i=0; i<xValues.length; i++) {
+    dividendSum += (xValues[i] - xMean) * (yValues[i] - yMean);
+    divisorSum += Math.pow(xValues[i] - xMean, 2);
+  }
+  return dividendSum / divisorSum;
+}

tools/benchpress/src/stats.es6

@@ -1,59 +0,0 @@
-module.exports = {
-  createObjectStatsAggregator: createObjectStatsAggregator,
-  calculateCoefficientOfVariation: calculateCoefficientOfVariation,
-  calculateMean: calculateMean,
-  calculateStandardDeviation: calculateStandardDeviation
-};
-
-function createObjectStatsAggregator(properties, sampleSize) {
-  var propSamples = {};
-  addData.current = {};
-  properties.forEach(function(prop) {
-    addData.current[prop] = {
-      mean: 0,
-      coefficientOfVariation: 0,
-      count: 0
-    };
-  });
-  return addData;
-
-  function addData(data) {
-    var result = {};
-    properties.forEach(function(prop) {
-      var samples = propSamples[prop];
-      if (!samples) {
-        samples = propSamples[prop] = [];
-      }
-      samples.push(data[prop]);
-      samples.splice(0, samples.length - sampleSize);
-      var mean = calculateMean(samples);
-      result[prop] = {
-        mean: mean,
-        coefficientOfVariation: calculateCoefficientOfVariation(samples, mean),
-        count: samples.length
-      };
-    });
-    addData.current = result;
-    return result;
-  }
-}
-
-function calculateCoefficientOfVariation(sample, mean) {
-  return calculateStandardDeviation(sample, mean) / mean * 100;
-}
-
-function calculateMean(sample) {
-  var total = 0;
-  sample.forEach(function(x) { total += x; });
-  return total / sample.length;
-}
-
-function calculateStandardDeviation(sample, mean) {
-  var deviation = 0;
-  sample.forEach(function(x) {
-    deviation += Math.pow(x - mean, 2);
-  });
-  deviation = deviation / (sample.length);
-  deviation = Math.sqrt(deviation);
-  return deviation;
-}

tools/build/snippets/runtime_paths.js

@@ -9,6 +9,7 @@ System.paths = {
   'reflection/*': '/reflection/lib/*.js',
   'benchpress/*': '/benchpress/lib/*.js',
   'examples/*': '/examples/web/*.js',
+  'e2e_test_lib/*': '/e2e_test_lib/lib/*.js',
   'benchmarks/*': '/benchmarks/web/*.js',
   'benchmarks_external/*': '/benchmarks_external/web/*.js',
 };

tools/build/snippets/url_params_to_form.js

@@ -0,0 +1,14 @@
+// helper script that will read out the url parameters
+// and store them in appropriate form fields on the page
+(function() {
+  var regex = /(\w+)=(\w+)/g;
+  var search = decodeURIComponent(location.search);
+  while (match = regex.exec(search)) {
+    var name = match[1];
+    var value = match[2];
+    var els = document.querySelectorAll('input[name="'+name+'"]');
+    for (var i=0; i<els.length; i++) {
+      els[i].value = value;
+    }
+  }
+})();