angular-cn/tools/perf/util.js

var webdriver = require('protractor/node_modules/selenium-webdriver');

module.exports = {
  perfLogs: perfLogs,
  sumTimelineRecords: sumTimelineRecords,
  runClickBenchmark: runClickBenchmark,
  verifyNoErrors: verifyNoErrors,
  printObjectAsMarkdown: printObjectAsMarkdown
};

function runClickBenchmark(config) {
  var buttons = config.buttons.map(function(selector) {
    return $(selector);
  });
  var globalParams = browser.params;

  // empty perflogs queue and gc
  gc();
  perfLogs();
  var sampleQueue = [];
  var bestSampleStats = null;

  loop(globalParams.maxRepeatCount).then(function(stats) {
    printObjectAsMarkdown(config.name, stats);
  });

  function loop(count) {
    if (!count) {
      return bestSampleStats;
    }
    return webdriver.promise.all(buttons.map(function(button) {
      // Note: even though we remove the gc time from the script time,
      // we still get a high standard devication if we don't gc after every click...
      return button.click().then(gc);
    })).then(function() {
      return perfLogs();
    }).then(function(logs) {
      var stats = calculateStatsBasedOnLogs(logs);
      if (stats) {
        if (stats.script.error < globalParams.exitOnErrorLowerThan) {
          return stats;
        }
        if (!bestSampleStats || stats.script.error < bestSampleStats.script.error) {
          bestSampleStats = stats;
        }
      }
      return loop(count-1);
    });
  }

  function calculateStatsBasedOnLogs(logs) {
    sampleQueue.push(sumTimelineRecords(logs['Timeline.eventRecorded']));
    if (sampleQueue.length >= globalParams.sampleSize) {
      sampleQueue.splice(0, sampleQueue.length - globalParams.sampleSize);
      // TODO: gc numbers don't have much meaning right now,
      // as a benchmark run destroys everything.
      // We need to measure the heap size after gc as well!
      return calculateObjectSampleStats(sampleQueue, ['script', 'render', 'gcTime', 'gcAmount']);
    }
    return null;
  }
}

function gc() {
  // TODO(tbosch): this only works on chrome, and we actually should
  // extend chromedriver to use the Debugger.CollectGarbage call of the
  // remote debugger protocol.
  // See http://src.chromium.org/viewvc/blink/trunk/Source/devtools/protocol.json
  // For iOS Safari we need an extension to appium that uses
  // the webkit remote debug protocol. See
  // https://github.com/WebKit/webkit/blob/master/Source/WebInspectorUI/Versions/Inspector-iOS-8.0.json
  return browser.executeScript('window.gc()');
}

function verifyNoErrors() {
  browser.manage().logs().get('browser').then(function(browserLog) {
    var filteredLog = browserLog.filter(function(logEntry) {
      return logEntry.level.value > webdriver.logging.Level.WARNING.value;
    });
    expect(filteredLog.length).toEqual(0);
    if (filteredLog.length) {
      console.log('browser console errors: ' + require('util').inspect(filteredLog));
    }
  });
}

function perfLogs() {
  return plainLogs('performance').then(function(entries) {
    var entriesByMethod = {};
    entries.forEach(function(entry) {
      var message = JSON.parse(entry.message).message;
      var entries = entriesByMethod[message.method];
      if (!entries) {
        entries = entriesByMethod[message.method] = [];
      }
      entries.push(message.params);
    });
    return entriesByMethod;
  });
}

// Needed as selenium-webdriver does not forward
// performance logs in the correct way
function plainLogs(type) {
  var webdriver = require('protractor/node_modules/selenium-webdriver');
  return browser.driver.schedule(
      new webdriver.Command(webdriver.CommandName.GET_LOG).
          setParameter('type', type),
      'WebDriver.manage().logs().get(' + type + ')');
};


function sumTimelineRecords(messages) {
  var recordStats = {
    script: 0,
    gcTime: 0,
    gcAmount: 0,
    render: 0
  };
  messages.forEach(function(message) {
    processRecord(message.record, recordStats);
  });
  return recordStats;

  function processRecord(record, recordStats) {
    var summedChildrenDuration = 0;
    if (record.children) {
      record.children.forEach(function(child) {
        summedChildrenDuration += processRecord(child, recordStats);
      });
    }

    var recordDuration;
    var recordUsed = false;
    if (recordStats) {
      // we need to substract the time of child records
      // that have been added to the stats from this record.
      // E.g. for a script record that triggered a gc or reflow while executing.
      recordDuration = (record.endTime ? record.endTime - record.startTime : 0)
        - summedChildrenDuration;
      if (record.type === 'FunctionCall') {
        if (!record.data || record.data.scriptName !== 'InjectedScript') {
          // ignore scripts that were injected by Webdriver (e.g. calculation of element positions, ...)
          recordStats.script += recordDuration;
          recordUsed = true;
        }
      } else if (record.type === 'GCEvent') {
        recordStats.gcTime += recordDuration;
        recordStats.gcAmount += record.data.usedHeapSizeDelta;
        recordUsed = true;
      } else if (record.type === 'RecalculateStyles' ||
          record.type === 'Layout' ||
          record.type === 'UpdateLayerTree' ||
          record.type === 'Paint' ||
          record.type === 'Rasterize' ||
          record.type === 'CompositeLayers') {
        recordStats.render += recordDuration;
        recordUsed = true;
      }
    }
    return recordUsed ? recordDuration : summedChildrenDuration;
  }
}

function printObjectAsMarkdown(name, obj) {
  var props = [['name']];
  var vals = [name];
  flattenObj(obj, [], props, vals);
  // log header
  var separators = [];
  var header = props.map(function(propPath) {
    separators.push('----');
    return propPath.join('.');
  }).join(' | ');
  console.log('\n'+header);
  console.log(separators.join(' | '));
  console.log(vals.join(' | '));
  console.log('\n');

  function flattenObj(obj, propPathPrefix, targetProps, targetVals) {
    for (var prop in obj) {
      var val = obj[prop];
      var currPropPath = propPathPrefix.concat([prop]);
      if (val && typeof val === 'object') {
        flattenObj(val, currPropPath, targetProps, targetVals);
      } else {
        targetProps.push(currPropPath);
        var valStr = val;
        if (typeof val === 'number') {
          valStr = val.toFixed(2);
        }
        targetVals.push(valStr);
      }
    }
  }
}

function calculateObjectSampleStats(objectSamples, properties) {
  var result = {};
  properties.forEach(function(prop) {
    var samples = objectSamples.map(function(objectSample) {
      return objectSample[prop];
    });
    var mean = calculateMean(samples);
    var error = calculateCoefficientOfVariation(samples, mean);
    result[prop] = {
      mean: mean,
      error: error
    };
  });
  return result;
}

function calculateCoefficientOfVariation(sample, mean) {
  return calculateStandardDeviation(sample, mean) / mean * 100;
}

function calculateMean(sample) {
  var total = 0;
  sample.forEach(function(x) { total += x; });
  return total / sample.length;
}

function calculateStandardDeviation(sample, mean) {
  var deviation = 0;
  sample.forEach(function(x) {
    deviation += Math.pow(x - mean, 2);
  });
  deviation = deviation / (sample.length -1);
  deviation = Math.sqrt(deviation);
  return deviation;
};