feat(perf): cloud reporter, more generic table layout

protractor-perf-shared.js

@@ -1,6 +1,7 @@
 var config = exports.config = require('./protractor-shared.js').config;
 // load traceur runtime as our tests are written in es6
 require('traceur/bin/traceur-runtime.js');
+var nodeUuid = require('node-uuid');
 
 var cloudReporterConfig;
 if (process.env.CLOUD_SECRET_PATH) {
@@ -8,15 +9,22 @@ if (process.env.CLOUD_SECRET_PATH) {
   cloudReporterConfig = {
     auth: require(process.env.CLOUD_SECRET_PATH),
     projectId: 'angular-perf',
-    datasetId: 'benchmarks'
+    datasetId: 'benchmarks',
+    tableId: 'ng2perf'
   };
 }
 
 config.specs = ['dist/cjs/**/*_perf.js'];
 config.jasmineNodeOpts.defaultTimeoutInterval = 80000;
 
+var runId = nodeUuid.v1();
+if (process.env.GIT_SHA) {
+  runId = process.env.GIT_SHA + ' ' + runId;
+}
+
 config.params = {
   benchmark: {
+    runId: runId,
     // size of the sample to take
     sampleSize: 20,
     timeout: 60000,

scripts/jenkins/jenkins_perf.sh

@@ -10,6 +10,7 @@ export CHANNEL=stable
 export ARCH=macos-ia32
 export PERF_BROWSERS=ChromeAndroid
 export CLOUD_SECRET_PATH="/Users/Shared/jenkins/keys/perf-cloud-secret"
+export GIT_SHA=$(git rev-parse HEAD)
 
 nvm use 0.10
 

tools/benchpress/src/benchmark.es6

@@ -1,6 +1,5 @@
 var statistics = require('./statistics');
 var commands = require('./commands');
-var nodeUuid = require('node-uuid');
 var webdriver = require('protractor/node_modules/selenium-webdriver');
 
 var SUPPORTED_METRICS = {
@@ -21,11 +20,10 @@ module.exports = {
 };
 
 function runBenchmark(config, workCallback) {
-  var sampleId = nodeUuid.v1();
   var reporters = config.reporters.filter(function(Class) {
     return !!Class;
   }).map(function(Class) {
-    return new Class(sampleId, config);
+    return new Class(config);
   });
   var scriptMetricIndex = -1;
   config.metrics.forEach(function(metric, index) {

tools/benchpress/src/cloud_reporter.es6

@@ -2,11 +2,16 @@ var google = require('googleapis');
 var bigquery = google.bigquery('v2');
 var webdriver = require('protractor/node_modules/selenium-webdriver');
 
-var HEADER_FIELDS = [
+var TABLE_FIELDS = [
   {
     "name": 'runId',
     "type": 'STRING',
-    "description": 'uuid for the benchmark run'
+    "description": 'git SHA and uuid for the benchmark run'
+  },
+  {
+    "name": 'benchmarkId',
+    "type": 'STRING',
+    "description": 'id of the benchmark'
   },
   {
     "name": 'index',
@@ -26,22 +31,65 @@ var HEADER_FIELDS = [
     "name": 'forceGc',
     "type": 'BOOLEAN',
     "description": 'whether gc was forced at end of action'
+  },
+  {
+    "name": 'stable',
+    "type": 'BOOLEAN',
+    "description": 'whether this entry was part of the stable sample'
+  },
+  {
+    "name": 'params',
+    "type": 'RECORD',
+    "description": 'parameters of the benchmark',
+    "mode": 'REPEATED',
+    "fields": [
+      {
+        "name": 'name',
+        "type": 'STRING',
+        "description": 'param name'
+      },
+      {
+        "name": 'strvalue',
+        "type": 'STRING',
+        "description": 'param value for strings'
+      },
+      {
+        "name": 'numvalue',
+        "type": 'FLOAT',
+        "description": 'param value for numbers'
+      }
+    ]
+  },
+  {
+    "name": 'metrics',
+    "type": 'RECORD',
+    "description": 'metrics of the benchmark',
+    "mode": 'REPEATED',
+    "fields": [
+      {
+        "name": 'name',
+        "type": 'STRING',
+        "description": 'metric name'
+      },
+      {
+        "name": 'value',
+        "type": 'FLOAT',
+        "description": 'metric value'
+      }
+    ]
   }
 ];
 
 class CloudReporter {
-  constructor(runId, benchmarkConfig) {
+  constructor(benchmarkConfig) {
-    this.stableRowsTableConfig = createTableConfig(benchmarkConfig, '_stable');
+    this.tableConfig = createTableConfig(benchmarkConfig);
-    this.allRowsTableConfig = createTableConfig(benchmarkConfig, '_all')
     this.authConfig = benchmarkConfig.cloudReporter.auth;
     this.benchmarkConfig = benchmarkConfig;
-    this.runId = runId;
+    this.allSample = [];
-    this.allRows = [];
     var self = this;
     browser.executeScript('return navigator.userAgent').then(function(userAgent) {
       self.browserUserAgent = userAgent;
     });
-
   }
   begin() {
     var self = this;
@@ -52,66 +100,62 @@ class CloudReporter {
       });
     });
     flow.execute(function() {
-      return webdriver.promise.all([
+      return getOrCreateTable(self.authClient, self.tableConfig);
-        getOrCreateTable(self.authClient, self.allRowsTableConfig),
-        getOrCreateTable(self.authClient, self.stableRowsTableConfig)
-      ]);
     });
   }
   add(data) {
-    this.allRows.push(this._convertToTableRow(data));
+    this.allSample.push(data);
   }
   end(stableSample) {
     var self = this;
     var flow = browser.driver.controlFlow();
-    var stableRows = stableSample.map(function(data) {
+    var allRows = this.allSample.map(function(data) {
-      return self._convertToTableRow(data);
+      return self._convertToTableRow(data, stableSample);
     });
     flow.execute(function() {
-      return webdriver.promise.all([
+      return insertRows(self.authClient, self.tableConfig, allRows)
-        insertRows(self.authClient, self.stableRowsTableConfig, stableRows),
-        insertRows(self.authClient, self.allRowsTableConfig, self.allRows)
-      ]);
     });
   }
-  _convertToTableRow(benchpressRow) {
+  _convertToTableRow(benchpressRow, stableSample) {
     var tableRow = {
-      runId: this.runId,
+      runId: this.benchmarkConfig.runId,
+      benchmarkId: this.benchmarkConfig.id,
       index: benchpressRow.index,
       creationTime: new Date(),
       browser: this.browserUserAgent,
-      forceGc: benchpressRow.forceGc
+      forceGc: benchpressRow.forceGc,
+      stable: stableSample.indexOf(benchpressRow) >= 0,
+      params: this.benchmarkConfig.params.map(function(param) {
+        if (typeof param.value === 'number') {
+          return {
+            name: param.name,
+            numvalue: param.value
+          };
+        } else {
+          return {
+            name: param.name,
+            strvalue: ''+param.value
+          }
+        }
+      }),
+      metrics: this.benchmarkConfig.metrics.map(function(metricName, index) {
+        return {
+          name: metricName,
+          value: benchpressRow.values[index]
+        };
+      })
     };
-    this.benchmarkConfig.params.forEach(function(param) {
-      tableRow['p_'+param.name] = param.value;
-    });
-    this.benchmarkConfig.metrics.forEach(function(metric, index) {
-      tableRow['m_'+metric] = benchpressRow.values[index];
-    });
     return tableRow;
   }
 }
 
-function createTableConfig(benchmarkConfig, tableSuffix) {
+function createTableConfig(benchmarkConfig) {
-  var tableId = (benchmarkConfig.id+tableSuffix).replace(/\./g, '_');
   return {
     projectId: benchmarkConfig.cloudReporter.projectId,
     datasetId: benchmarkConfig.cloudReporter.datasetId,
     table: {
-      id: tableId,
+      id: benchmarkConfig.cloudReporter.tableId,
-      fields: HEADER_FIELDS
+      fields: TABLE_FIELDS
-        .concat(benchmarkConfig.params.map(function(param) {
-          return {
-            "name": 'p_'+param.name,
-            "type": 'FLOAT'
-          };
-        }))
-        .concat(benchmarkConfig.metrics.map(function(metricName) {
-          return {
-            "name": 'm_'+metricName,
-            "type": 'FLOAT'
-          };
-        }))
     }
   };
 }

tools/benchpress/src/console_reporter.es6

@@ -5,9 +5,8 @@ var HEADER_SEPARATORS = ['----', '----', '----', '----', '----', '----', '----']
 var FOOTER_SEPARATORS = ['====', '====', '====', '====', '====', '====', '===='];
 
 class ConsoleReporter {
-  constructor(runId, config) {
+  constructor(config) {
     this.config = config;
-    this.runId = runId;
     this.rowFormat = ['%12s'].concat(config.metrics.map(function() {
       return '%12s';
     })).join(' | ');
@@ -15,7 +14,7 @@ class ConsoleReporter {
   begin() {
     printHeading('BENCHMARK '+this.config.id);
     console.log('sample size', this.config.sampleSize);
-    console.log('run id', this.runId);
+    console.log('run id', this.config.runId);
     console.log('params', JSON.stringify(this.config.params, null, '  '));
     printTableHeader(this.rowFormat, ['index', 'forceGc'].concat(this.config.metrics));
   }