feat(benchpress): show more metrics and make the run mode configurable

Shows the metrics:
script, render, gcAmount, gcAmountInScript, gcTime

Run modes:
- detect: auto detect whether to force gc
- forceGc: forces a gc before every run and ignores no runs
- noGcInScript: ignore runs that have gc while a script was executing
- plain: does not force gc nor ignore runs

Closes #368

modules/benchmarks/e2e_test/change_detection_perf.es6

@@ -29,14 +29,9 @@ function runClickBenchmark(config) {
   var buttons = config.buttons.map(function(selector) {
     return $(selector);
   });
-  var timeParams = browser.params.benchmark;
+  var params = Object.create(browser.params.benchmark);
-  benchpress.runBenchmark({
+  params.logId = browser.params.lang+'.'+config.logId;
-    sampleSize: timeParams.sampleSize,
+  benchpress.runBenchmark(params, function() {
-    targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
-    timeout: timeParams.timeout,
-    metrics: timeParams.metrics,
-    logId: browser.params.lang+'.'+config.logId
-  }, function() {
     buttons.forEach(function(button) {
       button.click();
     });

modules/benchmarks/e2e_test/compiler_perf.es6

@@ -29,14 +29,9 @@ function runClickBenchmark(config) {
   var buttons = config.buttons.map(function(selector) {
     return $(selector);
   });
-  var timeParams = browser.params.benchmark;
+  var params = Object.create(browser.params.benchmark);
-  benchpress.runBenchmark({
+  params.logId = browser.params.lang+'.'+config.logId;
-    sampleSize: timeParams.sampleSize,
+  benchpress.runBenchmark(params, function() {
-    targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
-    timeout: timeParams.timeout,
-    metrics: timeParams.metrics,
-    logId: browser.params.lang+'.'+config.logId
-  }, function() {
     buttons.forEach(function(button) {
       button.click();
     });

modules/benchmarks/e2e_test/di_perf.es6

@@ -45,14 +45,9 @@ function runClickBenchmark(config) {
   var buttons = config.buttons.map(function(selector) {
     return $(selector);
   });
-  var timeParams = browser.params.benchmark;
+  var params = Object.create(browser.params.benchmark);
-  benchpress.runBenchmark({
+  params.logId = browser.params.lang+'.'+config.logId;
-    sampleSize: timeParams.sampleSize,
+  benchpress.runBenchmark(params, function() {
-    targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
-    timeout: timeParams.timeout,
-    metrics: timeParams.metrics,
-    logId: browser.params.lang+'.'+config.logId
-  }, function() {
     buttons.forEach(function(button) {
       button.click();
     });

modules/benchmarks/e2e_test/element_injector_perf.es6

@@ -29,14 +29,9 @@ function runClickBenchmark(config) {
   var buttons = config.buttons.map(function(selector) {
     return $(selector);
   });
-  var timeParams = browser.params.benchmark;
+  var params = Object.create(browser.params.benchmark);
-  benchpress.runBenchmark({
+  params.logId = browser.params.lang+'.'+config.logId;
-    sampleSize: timeParams.sampleSize,
+  benchpress.runBenchmark(params, function() {
-    targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
-    timeout: timeParams.timeout,
-    metrics: timeParams.metrics,
-    logId: browser.params.lang+'.'+config.logId
-  }, function() {
     buttons.forEach(function(button) {
       button.click();
     });

modules/benchmarks/e2e_test/tree_perf.es6

@@ -29,14 +29,9 @@ function runClickBenchmark(config) {
   var buttons = config.buttons.map(function(selector) {
     return $(selector);
   });
-  var timeParams = browser.params.benchmark;
+  var params = Object.create(browser.params.benchmark);
-  benchpress.runBenchmark({
+  params.logId = browser.params.lang+'.'+config.logId;
-    sampleSize: timeParams.sampleSize,
+  benchpress.runBenchmark(params, function() {
-    targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
-    timeout: timeParams.timeout,
-    metrics: timeParams.metrics,
-    logId: browser.params.lang+'.'+config.logId
-  }, function() {
     buttons.forEach(function(button) {
       button.click();
     });

modules/benchmarks_external/e2e_test/compiler_perf.es6

@@ -29,14 +29,9 @@ function runClickBenchmark(config) {
   var buttons = config.buttons.map(function(selector) {
     return $(selector);
   });
-  var timeParams = browser.params.benchmark;
+  var params = Object.create(browser.params.benchmark);
-  benchpress.runBenchmark({
+  params.logId = browser.params.lang+'.'+config.logId;
-    sampleSize: timeParams.sampleSize,
+  benchpress.runBenchmark(params, function() {
-    targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
-    timeout: timeParams.timeout,
-    metrics: timeParams.metrics,
-    logId: browser.params.lang+'.'+config.logId
-  }, function() {
     buttons.forEach(function(button) {
       button.click();
     });

modules/benchmarks_external/e2e_test/tree_perf.es6

@@ -21,14 +21,9 @@ function runClickBenchmark(config) {
   var buttons = config.buttons.map(function(selector) {
     return $(selector);
   });
-  var timeParams = browser.params.benchmark;
+  var params = Object.create(browser.params.benchmark);
-  benchpress.runBenchmark({
+  params.logId = browser.params.lang+'.'+config.logId;
-    sampleSize: timeParams.sampleSize,
+  benchpress.runBenchmark(params, function() {
-    targetCoefficientOfVariation: timeParams.targetCoefficientOfVariation,
-    timeout: timeParams.timeout,
-    metrics: timeParams.metrics,
-    logId: browser.params.lang+'.'+config.logId
-  }, function() {
     buttons.forEach(function(button) {
       button.click();
     });

protractor-perf-shared.js

@@ -11,7 +11,13 @@ var config = exports.config = {
       sampleSize: 10,
       targetCoefficientOfVariation: 4,
       timeout: 20000,
-      metrics: ['script', 'render']
+      metrics: ['script', 'render', 'gcAmount', 'gcAmountInScript', 'gcTime'],
+      // run mode of the benchmark:
+      // - detect: auto detect whether to force gc
+      // - forceGc: forces a gc before every run and ignores no runs
+      // - noGcInScript: ignore runs that have gc while a script was executing
+      // - plain: does not force nor ignore runs
+      mode: 'detect'
     }
   },
 

tools/benchpress/src/benchmark.es6

@@ -6,16 +6,57 @@ var SUPPORTED_METRICS = {
   script: true,
   gcTime: true,
   gcAmount: true,
-  gcTimeDuringScript: true,
+  gcTimeInScript: true,
-  gcAmountDuringScript: true,
+  gcAmountInScript: true,
   gcAmountPerMs: true,
   render: true
 };
-var DETERMINE_FORCE_GC_MODE_ITERATIONS = 5;
 
-var MODE_FORCE_GC = 'forceGc';
+var RUN_MODE = {
-var MODE_IGNORE_RUNS_WITH_GC = 'ignoreRunsWithGc';
+  detect: function(prevState, benchmarkData, iterationIndex) {
-var MODE_INDETERMINATE = 'indeterminate';
+    var gcInScriptCount = prevState.gcInScriptCount || 0;
+    if (benchmarkData.gcAmountInScript) {
+      gcInScriptCount++;
+    }
+    var ignoreRun = !!benchmarkData.gcAmountInScript;
+    var nextMode = RUN_MODE.detect;
+    if (iterationIndex > 10) {
+      if (gcInScriptCount / iterationIndex > 0.7) {
+        nextMode = RUN_MODE.forceGc;
+      } else {
+        nextMode = RUN_MODE.noGcInScript;
+      }
+    }
+    return {
+      forceGc: false,
+      ignoreRun: ignoreRun,
+      gcInScriptCount: gcInScriptCount,
+      nextMode: nextMode
+    };
+  },
+  forceGc: function() {
+    return {
+      forceGc: true,
+      ignoreRun: false,
+      nextMode: RUN_MODE.forceGc
+    }
+  },
+  noGcInScript: function(prevState, benchmarkData) {
+    var ignoreRun = !!benchmarkData.gcAmountInScript;
+    return {
+      forceGc: false,
+      ignoreRun: ignoreRun,
+      nextMode: RUN_MODE.noGcInScript
+    }
+  },
+  plain: function() {
+    return {
+      forceGc: false,
+      ignoreRun: false,
+      nextMode: RUN_MODE.plain
+    }
+  }
+};
 
 var nextTimestampId = 0;
 
@@ -30,25 +71,30 @@ function runBenchmark(config, workCallback) {
       throw new Error('Metric '+metric+' is not suported by benchpress right now');
     }
   });
-  var ROW_FORMAT = ['%-40s'].concat(config.metrics.map(function() {
+  var ROW_FORMAT = ['%-40s', '%12s'].concat(config.metrics.map(function() {
     return '%12s';
   })).join(' | ');
 
-  var benchmarkStatsAggregator = stats.createObjectStatsAggregator(config.sampleSize);
+  var benchmarkStatsAggregator = stats.createObjectStatsAggregator(config.metrics, config.sampleSize);
 
   var startTime = Date.now();
   startLoop().then(endLoop);
 
-  var gcDuringScriptCount = 0;
-
   function startLoop(gcData) {
     reporter.printHeading('SCRIPT DATA: sampling size '+config.sampleSize);
-    reporter.printTableHeader(ROW_FORMAT, ['name'].concat(config.metrics));
+    reporter.printTableHeader(ROW_FORMAT, ['name', 'action'].concat(config.metrics));
-    return loop(0, MODE_INDETERMINATE);
+    if (!(config.mode in RUN_MODE)) {
+      throw new Error('Unknown mode '+config.mode);
+    }
+    return loop(0, {
+      forceGc: false,
+      ignoreRun: false,
+      nextMode: RUN_MODE[config.mode]
+    });
   }
 
   function endLoop(stats) {
-    reporter.printTableFooter(ROW_FORMAT, [config.logId]
+    reporter.printTableFooter(ROW_FORMAT, [config.logId, '']
       .concat(formatObjectStats(stats, config.metrics))
     );
     return config.metrics.map(function(metric) {
@@ -56,40 +102,35 @@ function runBenchmark(config, workCallback) {
     });
   }
 
-  function loop(iterationIndex, mode) {
+  function loop(iterationIndex, modeState) {
+    return measureTime(function() {
+      workCallback();
+      if (modeState.forceGc) {
         // For fast tests that don't create a lot of garbage,
         // we don't want to force gc before every run as that
         // can slow down the script execution time (even when we subtract
         // the gc time)!
-    if (mode === MODE_FORCE_GC) {
+        // Note: we need to call gc AFTER the actual test so the
+        // gc amount is added to the current test run!
         commands.gc();
       }
-    return measureTime(workCallback).then(function(benchmarkData) {
+    }).then(function(benchmarkData) {
-      var hasGcDuringScript = !!benchmarkData.gcTimeDuringScript;
+      modeState = modeState.nextMode(modeState, benchmarkData, iterationIndex);
-      var ignoreBenchmarkRun = false;
+      var action = '';
-      if (hasGcDuringScript) {
+      if (modeState.ignoreRun) {
-        gcDuringScriptCount ++;
+        action = 'ignore';
-        ignoreBenchmarkRun = (mode === MODE_INDETERMINATE || mode === MODE_IGNORE_RUNS_WITH_GC);
+      } else if (modeState.forceGc) {
+        action = 'forceGc';
       }
-      if (mode === MODE_INDETERMINATE && iterationIndex >= DETERMINE_FORCE_GC_MODE_ITERATIONS) {
+      reporter.printRow(ROW_FORMAT, [config.logId + '#' + iterationIndex, action]
-        mode = (gcDuringScriptCount / iterationIndex > 0.5) ? MODE_FORCE_GC : MODE_IGNORE_RUNS_WITH_GC;
-      }
-
-      var rowTitle;
-      if (ignoreBenchmarkRun) {
-        rowTitle = '(ignored: gc in script)';
-      } else {
-        rowTitle = config.logId + '#' + iterationIndex;
-      }
-      reporter.printRow(ROW_FORMAT, [rowTitle]
         .concat(formatObjectData(benchmarkData, config.metrics))
       );
 
       var benchmarkStats;
-      if (!ignoreBenchmarkRun) {
+      if (modeState.ignoreRun) {
-        benchmarkStats = benchmarkStatsAggregator(benchmarkData);
-      } else {
         benchmarkStats = benchmarkStatsAggregator.current;
+      } else {
+        benchmarkStats = benchmarkStatsAggregator(benchmarkData);
       }
 
       if (Date.now() - startTime > config.timeout) {
@@ -102,22 +143,26 @@ function runBenchmark(config, workCallback) {
         ) {
         return benchmarkStats
       }
-      return loop(iterationIndex+1, mode);
+      return loop(iterationIndex+1, modeState);
     });
   }
 }
 
-
 function formatObjectData(data, props) {
   return props.map(function(prop) {
-    return data[prop].toFixed(2);
+    var val = data[prop];
+    if (typeof val === 'number') {
+      return val.toFixed(2);
+    } else {
+      return val;
+    }
   });
 }
 
 function formatObjectStats(stats, props) {
   return props.map(function(prop) {
     var entry = stats[prop];
-    return entry.mean.toFixed(2) + '\u00B1' + entry.coefficientOfVariation.toFixed(2);
+    return entry.mean.toFixed(2) + '\u00B1' + entry.coefficientOfVariation.toFixed(0)+ '%';
   });
 }
 
@@ -156,8 +201,8 @@ function sumTimelineRecords(records, startTimeStampId, endTimeStampId) {
     script: 0,
     gcTime: 0,
     gcAmount: 0,
-    gcTimeDuringScript: 0,
+    gcTimeInScript: 0,
-    gcAmountDuringScript: 0,
+    gcAmountInScript: 0,
     render: 0,
     timeStamps: []
   };
@@ -205,8 +250,8 @@ function sumTimelineRecords(records, startTimeStampId, endTimeStampId) {
         recordStats.gcTime += recordDuration;
         recordStats.gcAmount += record.data.usedHeapSizeDelta;
         if (parentIsFunctionCall) {
-          recordStats.gcTimeDuringScript += recordDuration;
+          recordStats.gcTimeInScript += recordDuration;
-          recordStats.gcAmountDuringScript += record.data.usedHeapSizeDelta;
+          recordStats.gcAmountInScript += record.data.usedHeapSizeDelta;
         }
         recordUsed = true;
       } else if (record.type === 'RecalculateStyles' ||

tools/benchpress/src/reporter.es6

@@ -1,7 +1,7 @@
 var vsprintf = require("sprintf-js").vsprintf;
 
-var HEADER_SEPARATORS = ['----', '----', '----', '----'];
+var HEADER_SEPARATORS = ['----', '----', '----', '----', '----', '----', '----'];
-var FOOTER_SEPARATORS = ['====', '====', '====', '===='];
+var FOOTER_SEPARATORS = ['====', '====', '====', '====', '====', '====', '===='];
 
 module.exports = {
   printHeading: printHeading,

tools/benchpress/src/stats.es6

@@ -5,14 +5,21 @@ module.exports = {
   calculateStandardDeviation: calculateStandardDeviation
 };
 
-function createObjectStatsAggregator(sampleSize) {
+function createObjectStatsAggregator(properties, sampleSize) {
   var propSamples = {};
-  var lastResult;
+  addData.current = {};
+  properties.forEach(function(prop) {
+    addData.current[prop] = {
+      mean: 0,
+      coefficientOfVariation: 0,
+      count: 0
+    };
+  });
   return addData;
 
   function addData(data) {
-    lastResult = {};
+    var result = {};
-    for (var prop in data) {
+    properties.forEach(function(prop) {
       var samples = propSamples[prop];
       if (!samples) {
         samples = propSamples[prop] = [];
@@ -20,14 +27,14 @@ function createObjectStatsAggregator(sampleSize) {
       samples.push(data[prop]);
       samples.splice(0, samples.length - sampleSize);
       var mean = calculateMean(samples);
-      lastResult[prop] = {
+      result[prop] = {
         mean: mean,
         coefficientOfVariation: calculateCoefficientOfVariation(samples, mean),
         count: samples.length
       };
-    }
+    });
-    addData.current = lastResult;
+    addData.current = result;
-    return lastResult;
+    return result;
   }
 }