From 5f5ed0671337798d9f2a5ee815a0ecf2f1bb8155 Mon Sep 17 00:00:00 2001 From: Tobias Bosch Date: Fri, 16 Jan 2015 17:36:24 -0800 Subject: [PATCH] feat(perf): cloud reporter should retry in case of a timeout --- tools/benchpress/src/cloud_reporter.es6 | 157 +++++++++++++++--------- 1 file changed, 97 insertions(+), 60 deletions(-) diff --git a/tools/benchpress/src/cloud_reporter.es6 b/tools/benchpress/src/cloud_reporter.es6 index 58fe0f9993..9020c0b4b5 100644 --- a/tools/benchpress/src/cloud_reporter.es6 +++ b/tools/benchpress/src/cloud_reporter.es6 @@ -80,6 +80,8 @@ var TABLE_FIELDS = [ } ]; +var RETRY_COUNT = 3; + class CloudReporter { constructor(benchmarkConfig) { this.tableConfig = createTableConfig(benchmarkConfig); @@ -95,12 +97,12 @@ class CloudReporter { var self = this; var flow = browser.driver.controlFlow(); flow.execute(function() { - return authenticate(self.authConfig).then(function(authClient) { + return authenticate(self.authConfig, RETRY_COUNT).then(function(authClient) { self.authClient = authClient; }); }); flow.execute(function() { - return getOrCreateTable(self.authClient, self.tableConfig); + return getOrCreateTable(self.authClient, self.tableConfig, RETRY_COUNT); }); } add(data) { @@ -112,40 +114,40 @@ class CloudReporter { var allRows = this.allSample.map(function(data) { return self._convertToTableRow(data, stableSample); }); - flow.execute(function() { - return insertRows(self.authClient, self.tableConfig, allRows) - }); + return insertRows(this.authClient, this.tableConfig, allRows, RETRY_COUNT); } _convertToTableRow(benchpressRow, stableSample) { - var tableRow = { - runId: this.benchmarkConfig.runId, - benchmarkId: this.benchmarkConfig.id, - index: benchpressRow.index, - creationTime: new Date(), - browser: this.browserUserAgent, - forceGc: benchpressRow.forceGc, - stable: stableSample.indexOf(benchpressRow) >= 0, - params: this.benchmarkConfig.params.map(function(param) { - if (typeof param.value === 'number') { - return { - name: param.name, - numvalue: param.value - }; - } else { - return { - name: param.name, - strvalue: ''+param.value + return { + insertId: this.benchmarkConfig.runId+'#'+benchpressRow.index, + json: { + runId: this.benchmarkConfig.runId, + benchmarkId: this.benchmarkConfig.id, + index: benchpressRow.index, + creationTime: new Date(), + browser: this.browserUserAgent, + forceGc: benchpressRow.forceGc, + stable: stableSample.indexOf(benchpressRow) >= 0, + params: this.benchmarkConfig.params.map(function(param) { + if (typeof param.value === 'number') { + return { + name: param.name, + numvalue: param.value + }; + } else { + return { + name: param.name, + strvalue: ''+param.value + } } - } - }), - metrics: this.benchmarkConfig.metrics.map(function(metricName, index) { - return { - name: metricName, - value: benchpressRow.values[index] - }; - }) + }), + metrics: this.benchmarkConfig.metrics.map(function(metricName, index) { + return { + name: metricName, + value: benchpressRow.values[index] + }; + }) + } }; - return tableRow; } } @@ -160,14 +162,14 @@ function createTableConfig(benchmarkConfig) { }; } -function getOrCreateTable(authClient, tableConfig) { - return getTable(authClient, tableConfig).then(null, function(err) { +function getOrCreateTable(authClient, tableConfig, retryCount) { + return getTable(authClient, tableConfig, retryCount).then(null, function(err) { // create the table if it does not exist - return createTable(authClient, tableConfig); + return createTable(authClient, tableConfig, retryCount); }); } -function authenticate(authConfig) { +function authenticate(authConfig, retryCount) { var authClient = new google.auth.JWT( authConfig['client_email'], null, @@ -178,12 +180,16 @@ function authenticate(authConfig) { var defer = webdriver.promise.defer(); authClient.authorize(makeNodeJsResolver(defer)); - return defer.promise.then(function() { + var resultPromise = defer.promise.then(function() { return authClient; }); + resultPromise = retryIfNeeded(resultPromise, retryCount, function(newRetryCount) { + return authenticate(authConfig, newRetryCount); + }); + return resultPromise; } -function getTable(authClient, tableConfig) { +function getTable(authClient, tableConfig, retryCount) { // see https://cloud.google.com/bigquery/docs/reference/v2/tables/get var params = { auth: authClient, @@ -193,10 +199,14 @@ function getTable(authClient, tableConfig) { }; var defer = webdriver.promise.defer(); bigquery.tables.get(params, makeNodeJsResolver(defer)); - return defer.promise; + var resultPromise = defer.promise; + resultPromise = retryIfNeeded(resultPromise, retryCount, function(newRetryCount) { + return getTable(authClient, tableConfig, newRetryCount); + }); + return resultPromise; } -function createTable(authClient, tableConfig) { +function createTable(authClient, tableConfig, retryCount) { // see https://cloud.google.com/bigquery/docs/reference/v2/tables // see https://cloud.google.com/bigquery/docs/reference/v2/tables#resource var params = { @@ -217,10 +227,23 @@ function createTable(authClient, tableConfig) { }; var defer = webdriver.promise.defer(); bigquery.tables.insert(params, makeNodeJsResolver(defer)); - return defer.promise; + var resultPromise = defer.promise; + resultPromise = retryIfNeeded(resultPromise, retryCount, function(newRetryCount) { + return createTable(authClient, tableConfig, newRetryCount); + }); + return resultPromise; } -function insertRows(authClient, tableConfig, rows) { +function insertRows(authClient, tableConfig, rows, retryCount) { + // We need to split up the rows in batches as BigQuery + // has a size limit on requests. + // Note: executing the requests in parallel leads to timeouts sometime... + var recurseRows = null; + if (rows.length > 10) { + recurseRows = rows.slice(10); + rows = rows.slice(0, 10); + } + // see https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll var params = { auth: authClient, @@ -229,22 +252,41 @@ function insertRows(authClient, tableConfig, rows) { tableId: tableConfig.table.id, resource: { "kind": "bigquery#tableDataInsertAllRequest", - "rows": rows.map(function(row) { - return { - json: row - } - }) + "rows": rows } }; var defer = webdriver.promise.defer(); bigquery.tabledata.insertAll(params, makeNodeJsResolver(defer)); - return defer.promise.then(function(result) { + var resultPromise = defer.promise.then(function(result) { if (result.insertErrors) { - throw result.insertErrors.map(function(err) { - return err.errors.map(function(err) { - return err.message; - }).join('\n'); - }).join('\n'); + throw JSON.stringify(result.insertErrors, null, ' '); + } + }); + resultPromise = retryIfNeeded(resultPromise, retryCount, function(newRetryCount) { + return insertRows(authClient, tableConfig, rows, newRetryCount); + }); + if (recurseRows) { + resultPromise = resultPromise.then(function() { + return insertRows(authClient, tableConfig, recurseRows, retryCount); + }); + } + return resultPromise; +} + +function retryIfNeeded(promise, retryCount, retryCallback) { + if (!retryCount) { + return promise; + } + return promise.then(null, function(err) { + var errStr = err.toString(); + if (typeof err === 'object') { + errStr += JSON.stringify(err, null, ' '); + } + if (errStr.indexOf('timeout') !== -1) { + console.log('Retrying', retryCallback.toString()); + return retryCallback(); + } else { + throw err; } }); } @@ -252,13 +294,8 @@ function insertRows(authClient, tableConfig, rows) { function makeNodeJsResolver(defer) { return function(err, result) { if (err) { - // Normalize errors messages from BigCloud so that they show up nicely - if (err.errors) { - err = err.errors.map(function(err) { - return err.message; - }).join('\n'); - } - defer.reject(err); + // Format errors in a nice way + defer.reject(JSON.stringify(err, null, ' ')); } else { defer.fulfill(result); }