build(broccoli): add DiffingBroccoliPlugin and refactor existing plugins to use it

tree-differ:
 - export both TreeDiffer and DiffResult  interface

 diffing-broccoli-plugin:
 - factory class for wrapping DiffingBroccoliPlugins and turning them into BroccoliTrees

 broccoli-dest-copy:
 - refactor into DiffingBroccoliPlugin

 broccoli-traceur:
 - refactor into DiffingBroccoliPlugin

tools/broccoli/broccoli-dest-copy.ts

@@ -1,40 +1,21 @@
-/// <reference path="./broccoli.d.ts" />
 /// <reference path="../typings/node/node.d.ts" />
 /// <reference path="../typings/fs-extra/fs-extra.d.ts" />
 
 import fs = require('fs');
 import fse = require('fs-extra');
 import path = require('path');
-import TreeDiffer = require('./tree-differ');
+import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin';
 
 /**
  * Intercepts each file as it is copied to the destination tempdir,
  * and tees a copy to the given path outside the tmp dir.
  */
-export = function destCopy(inputTree, outputRoot) { return new DestCopy(inputTree, outputRoot); }
+class DestCopy implements DiffingBroccoliPlugin {
+  constructor(private inputPath, private cachePath, private outputRoot: string) {}
 
 
-class DestCopy implements BroccoliTree {
+  rebuild(treeDiff: DiffResult) {
-  treeDirtyChecker: TreeDiffer;
+    treeDiff.changedPaths.forEach((changedFilePath) => {
-  initialized = false;
-
-  // props monkey-patched by broccoli builder:
-  inputPath = null;
-  cachePath = null;
-  outputPath = null;
-
-
-  constructor(public inputTree: BroccoliTree, public outputRoot: string) {}
-
-
-  rebuild() {
-    let firstRun = !this.initialized;
-    this.init();
-
-    let diffResult = this.treeDirtyChecker.diffTree();
-    diffResult.log(!firstRun);
-
-    diffResult.changedPaths.forEach((changedFilePath) => {
       var destFilePath = path.join(this.outputRoot, changedFilePath);
 
       var destDirPath = path.dirname(destFilePath);
@@ -42,22 +23,13 @@ class DestCopy implements BroccoliTree {
       fse.copySync(path.join(this.inputPath, changedFilePath), destFilePath);
     });
 
-    diffResult.removedPaths.forEach((removedFilePath) => {
+    treeDiff.removedPaths.forEach((removedFilePath) => {
       var destFilePath = path.join(this.outputRoot, removedFilePath);
 
       // TODO: what about obsolete directories? we are not cleaning those up yet
       fs.unlinkSync(destFilePath);
     });
   }
-
-
-  private init() {
-    if (!this.initialized) {
-      this.initialized = true;
-      this.treeDirtyChecker = new TreeDiffer(this.inputPath);
-    }
-  }
-
-
-  cleanup() {}
 }
+
+export default wrapDiffingPlugin(DestCopy);

tools/broccoli/diffing-broccoli-plugin.ts

@@ -0,0 +1,103 @@
+/// <reference path="broccoli.d.ts" />
+/// <reference path="../typings/fs-extra/fs-extra.d.ts" />
+/// <reference path="../typings/node/node.d.ts" />
+
+import fs = require('fs');
+import fse = require('fs-extra');
+import path = require('path');
+import {TreeDiffer, DiffResult} from './tree-differ';
+let symlinkOrCopy = require('symlink-or-copy');
+
+
+export {DiffResult} from './tree-differ';
+
+
+/**
+ * Makes writing diffing plugins easy.
+ *
+ * Factory method that takes a class that implements the DiffingBroccoliPlugin interface and returns
+ * an instance of BroccoliTree.
+ *
+ * @param pluginClass
+ * @returns {DiffingPlugin}
+ */
+export function wrapDiffingPlugin(pluginClass): DiffingPluginWrapperFactory {
+  return function() { return new DiffingPluginWrapper(pluginClass, arguments); };
+}
+
+
+export interface DiffingBroccoliPlugin {
+  rebuild(diff: DiffResult): (Promise<any>| void);
+  cleanup ? () : void;
+}
+
+
+type DiffingPluginWrapperFactory = (inputTrees: (BroccoliTree | BroccoliTree[]), options?) =>
+    BroccoliTree;
+
+
+class DiffingPluginWrapper implements BroccoliTree {
+  treeDiffer: TreeDiffer;
+  initialized = false;
+  wrappedPlugin: DiffingBroccoliPlugin = null;
+  inputTree = null;
+  inputTrees = null;
+  description = null;
+
+  // props monkey-patched by broccoli builder:
+  inputPath = null;
+  cachePath = null;
+  outputPath = null;
+
+
+  constructor(private pluginClass, private wrappedPluginArguments) {
+    if (Array.isArray(wrappedPluginArguments[0])) {
+      this.inputTrees = wrappedPluginArguments[0];
+    } else {
+      this.inputTree = wrappedPluginArguments[0];
+    }
+
+    this.description = this.pluginClass.name;
+  }
+
+
+  rebuild() {
+    let firstRun = !this.initialized;
+    this.init();
+
+    let diffResult = this.treeDiffer.diffTree();
+    diffResult.log(!firstRun);
+
+    var rebuildPromise = this.wrappedPlugin.rebuild(diffResult);
+
+    if (rebuildPromise) {
+      return (<Promise<any>>rebuildPromise).then(this.relinkOutputAndCachePaths.bind(this));
+    }
+
+    this.relinkOutputAndCachePaths();
+  }
+
+
+  private relinkOutputAndCachePaths() {
+    // just symlink the cache and output tree
+    fs.rmdirSync(this.outputPath);
+    symlinkOrCopy.sync(this.cachePath, this.outputPath);
+  }
+
+
+  private init() {
+    if (!this.initialized) {
+      this.initialized = true;
+      this.treeDiffer = new TreeDiffer(this.inputPath);
+      this.wrappedPlugin =
+          new this.pluginClass(this.inputPath, this.cachePath, this.wrappedPluginArguments[1]);
+    }
+  }
+
+
+  cleanup() {
+    if (this.wrappedPlugin.cleanup) {
+      this.wrappedPlugin.cleanup();
+    }
+  }
+}

tools/broccoli/traceur/index.ts

@@ -1,66 +1,38 @@
-/// <reference path="../broccoli.d.ts" />
-/// <reference path="../broccoli-writer.d.ts" />
 /// <reference path="../../typings/fs-extra/fs-extra.d.ts" />
 /// <reference path="../../typings/node/node.d.ts" />
 
 import fs = require('fs');
 import fse = require('fs-extra');
 import path = require('path');
-import TreeDiffer = require('../tree-differ');
+import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from '../diffing-broccoli-plugin';
-import Writer = require('broccoli-writer');
 
 let traceur = require('../../../../tools/transpiler');
-let symlinkOrCopy = require('symlink-or-copy');
 let xtend = require('xtend');
 
 
-export = traceurCompiler;
+class DiffingTraceurCompiler implements DiffingBroccoliPlugin {
-
+  constructor(public inputPath: string, public cachePath: string, public options) {}
-function traceurCompiler(inputTree, destExtension, destSourceMapExtension, options) {
-  return new TraceurCompiler(inputTree, destExtension, destSourceMapExtension, options);
-}
-
-traceurCompiler['RUNTIME_PATH'] = traceur.RUNTIME_PATH;
 
 
-class TraceurCompiler implements BroccoliTree {
+  rebuild(treeDiff: DiffResult) {
-  treeDiffer: TreeDiffer;
+    treeDiff.changedPaths.forEach((changedFilePath) => {
-  initialized = false;
-
-  // props monkey-patched by broccoli builder:
-  inputPath = null;
-  cachePath = null;
-  outputPath = null;
-
-
-  constructor(public inputTree: BroccoliTree, private destExtension: string,
-              private destSourceMapExtension: string, private options: {[key: string]: any}) {}
-
-
-  rebuild() {
-    let firstRun = !this.initialized;
-    this.init();
-
-    let diffResult = this.treeDiffer.diffTree();
-    diffResult.log(!firstRun);
-
-    diffResult.changedPaths.forEach((changedFilePath) => {
       var extension = path.extname(changedFilePath).toLowerCase();
       if (extension === '.js' || extension === '.es6' || extension === '.cjs') {
-        var options = xtend({filename: changedFilePath}, this.options);
+        var traceurOpts = xtend({filename: changedFilePath}, this.options.traceurOptions);
 
         var fsOpts = {encoding: 'utf-8'};
         var absoluteInputFilePath = path.join(this.inputPath, changedFilePath);
         var sourcecode = fs.readFileSync(absoluteInputFilePath, fsOpts);
 
-        var result = traceur.compile(options, changedFilePath, sourcecode);
+        var result = traceur.compile(traceurOpts, changedFilePath, sourcecode);
 
         // TODO: we should fix the sourceMappingURL written by Traceur instead of overriding
         // (but we might switch to typescript first)
-        var mapFilepath = changedFilePath.replace(/\.\w+$/, '') + this.destSourceMapExtension;
+        var mapFilepath =
+            changedFilePath.replace(/\.\w+$/, '') + this.options.destSourceMapExtension;
         result.js = result.js + '\n//# sourceMappingURL=./' + path.basename(mapFilepath);
 
-        var destFilepath = changedFilePath.replace(/\.\w+$/, this.destExtension);
+        var destFilepath = changedFilePath.replace(/\.\w+$/, this.options.destExtension);
         var destFile = path.join(this.cachePath, destFilepath);
         fse.mkdirsSync(path.dirname(destFile));
         fs.writeFileSync(destFile, result.js, fsOpts);
@@ -71,25 +43,15 @@ class TraceurCompiler implements BroccoliTree {
       }
     });
 
-    diffResult.removedPaths.forEach((removedFilePath) => {
+    treeDiff.removedPaths.forEach((removedFilePath) => {
-      var destFilepath = removedFilePath.replace(/\.\w+$/, this.destExtension);
+      var destFilepath = removedFilePath.replace(/\.\w+$/, this.options.destExtension);
       var absoluteOuputFilePath = path.join(this.cachePath, destFilepath);
       fs.unlinkSync(absoluteOuputFilePath);
     });
-
-    // just symlink the cache and output tree
-    fs.rmdirSync(this.outputPath);
-    symlinkOrCopy.sync(this.cachePath, this.outputPath);
   }
-
-
-  private init() {
-    if (!this.initialized) {
-      this.initialized = true;
-      this.treeDiffer = new TreeDiffer(this.inputPath);
-    }
-  }
-
-
-  cleanup() {}
 }
+
+let transpileWithTraceur = wrapDiffingPlugin(DiffingTraceurCompiler);
+let TRACEUR_RUNTIME_PATH = traceur.RUNTIME_PATH;
+
+export {transpileWithTraceur as default, TRACEUR_RUNTIME_PATH};

tools/broccoli/tree-differ.spec.ts

@@ -3,7 +3,7 @@
 
 let mockfs = require('mock-fs');
 import fs = require('fs');
-import TreeDiffer = require('./tree-differ');
+import {TreeDiffer} from './tree-differ';
 
 
 describe('TreeDiffer', () => {

tools/broccoli/tree-differ.ts

@@ -4,9 +4,7 @@ import fs = require('fs');
 import path = require('path');
 
 
-export = TreeDiffer;
+export class TreeDiffer {
-
-class TreeDiffer {
   private fingerprints: {[key: string]: string} = Object.create(null);
   private nextFingerprints: {[key: string]: string} = Object.create(null);
   private rootDirName: string;
@@ -15,7 +13,7 @@ class TreeDiffer {
 
 
   public diffTree(): DiffResult {
-    let result = new DiffResult(this.rootDirName);
+    let result = new DirtyCheckingDiffResult(this.rootDirName);
     this.dirtyCheckPath(this.rootPath, result);
     this.detectDeletionsAndUpdateFingerprints(result);
     result.endTime = Date.now();
@@ -23,7 +21,7 @@ class TreeDiffer {
   }
 
 
-  private dirtyCheckPath(rootDir: string, result: DiffResult) {
+  private dirtyCheckPath(rootDir: string, result: DirtyCheckingDiffResult) {
     fs.readdirSync(rootDir).forEach((segment) => {
       let absolutePath = path.join(rootDir, segment);
       let pathStat = fs.statSync(absolutePath);
@@ -76,7 +74,15 @@ class TreeDiffer {
 }
 
 
-class DiffResult {
+export interface DiffResult {
+  changedPaths: string[];
+  removedPaths: string[];
+  log(verbose: boolean): void;
+  toString(): string;
+}
+
+
+class DirtyCheckingDiffResult {
   public filesChecked: number = 0;
   public directoriesChecked: number = 0;
   public changedPaths: string[] = [];

tools/broccoli/trees/browser_tree.ts

@@ -1,6 +1,5 @@
 'use strict';
 
-var destCopy = require('../broccoli-dest-copy');
 var Funnel = require('broccoli-funnel');
 var flatten = require('broccoli-flatten');
 var htmlReplace = require('../html-replace');
@@ -9,9 +8,11 @@ var path = require('path');
 var replace = require('broccoli-replace');
 var stew = require('broccoli-stew');
 var ts2dart = require('../broccoli-ts2dart');
-var traceurCompiler = require('../traceur');
 var TypescriptCompiler = require('../typescript');
 
+import destCopy from '../broccoli-dest-copy';
+import {default as transpileWithTraceur, TRACEUR_RUNTIME_PATH} from '../traceur/index';
+
 
 var projectRootDir = path.normalize(path.join(__dirname, '..', '..', '..', '..'));
 
@@ -22,16 +23,20 @@ module.exports = function makeBrowserTree(options, destinationPath) {
       {include: ['**/**'], exclude: ['**/*.cjs', 'benchmarks/e2e_test/**'], destDir: '/'});
 
   // Use Traceur to transpile *.js sources to ES6
-  var traceurTree = traceurCompiler(modulesTree, '.es6', '.map', {
+  var traceurTree = transpileWithTraceur(modulesTree, {
-    sourceMaps: true,
+    destExtension: '.es6',
-    annotations: true,      // parse annotations
+    destSourceMapExtension: '.map',
-    types: true,            // parse types
+    traceurOptions: {
-    script: false,          // parse as a module
+      sourceMaps: true,
-    memberVariables: true,  // parse class fields
+      annotations: true,      // parse annotations
-    modules: 'instantiate',
+      types: true,            // parse types
-    // typeAssertionModule: 'rtts_assert/rtts_assert',
+      script: false,          // parse as a module
-    // typeAssertions: options.typeAssertions,
+      memberVariables: true,  // parse class fields
-    outputLanguage: 'es6'
+      modules: 'instantiate',
+      // typeAssertionModule: 'rtts_assert/rtts_assert',
+      // typeAssertions: options.typeAssertions,
+      outputLanguage: 'es6'
+    }
   });
 
   // Use TypeScript to transpile the *.ts files to ES6
@@ -52,8 +57,11 @@ module.exports = function makeBrowserTree(options, destinationPath) {
   var es6Tree = mergeTrees([traceurTree, typescriptTree]);
 
   // Call Traceur again to lower the ES6 build tree to ES5
-  var es5Tree =
+  var es5Tree = transpileWithTraceur(es6Tree, {
-      traceurCompiler(es6Tree, '.js', '.js.map', {modules: 'instantiate', sourceMaps: true});
+    destExtension: '.js',
+    destSourceMapExtension: '.js.map',
+    traceurOptions: {modules: 'instantiate', sourceMaps: true}
+  });
 
   // Now we add a few more files to the es6 tree that Traceur should not see
   ['angular2', 'rtts_assert'].forEach(function(destDir) {
@@ -73,7 +81,7 @@ module.exports = function makeBrowserTree(options, destinationPath) {
       'node_modules/rx/dist/rx.js',
       'node_modules/reflect-metadata/Reflect.js',
       'tools/build/snippets/runtime_paths.js',
-      path.relative(projectRootDir, traceurCompiler.RUNTIME_PATH)
+      path.relative(projectRootDir, TRACEUR_RUNTIME_PATH)
     ]
   }));
   var vendorScripts_benchmark =

tools/broccoli/trees/dart_tree.ts

@@ -2,7 +2,7 @@
 'use strict';
 
 import {MultiCopy} from './../multi_copy';
-var destCopy = require('../broccoli-dest-copy');
+import destCopy from '../broccoli-dest-copy';
 var Funnel = require('broccoli-funnel');
 var glob = require('glob');
 var mergeTrees = require('broccoli-merge-trees');

tools/broccoli/trees/node_tree.ts

@@ -1,6 +1,6 @@
 'use strict';
 
-var destCopy = require('../broccoli-dest-copy');
+import destCopy from '../broccoli-dest-copy';
 var Funnel = require('broccoli-funnel');
 var mergeTrees = require('broccoli-merge-trees');
 var path = require('path');
@@ -8,7 +8,7 @@ var renderLodashTemplate = require('broccoli-lodash');
 var replace = require('broccoli-replace');
 var stew = require('broccoli-stew');
 var ts2dart = require('../broccoli-ts2dart');
-var traceurCompiler = require('../traceur');
+import transpileWithTraceur from '../traceur/index';
 var TypescriptCompiler = require('../typescript');
 
 var projectRootDir = path.normalize(path.join(__dirname, '..', '..', '..', '..'));
@@ -27,16 +27,20 @@ module.exports = function makeNodeTree(destinationPath) {
     ]
   });
 
-  var nodeTree = traceurCompiler(modulesTree, '.js', '.map', {
+  var nodeTree = transpileWithTraceur(modulesTree, {
-    sourceMaps: true,
+    destExtension: '.js',
-    annotations: true,      // parse annotations
+    destSourceMapExtension: '.map',
-    types: true,            // parse types
+    traceurOptions: {
-    script: false,          // parse as a module
+      sourceMaps: true,
-    memberVariables: true,  // parse class fields
+      annotations: true,      // parse annotations
-    typeAssertionModule: 'rtts_assert/rtts_assert',
+      types: true,            // parse types
-    // Don't use type assertions since this is partly transpiled by typescript
+      script: false,          // parse as a module
-    typeAssertions: false,
+      memberVariables: true,  // parse class fields
-    modules: 'commonjs'
+      typeAssertionModule: 'rtts_assert/rtts_assert',
+      // Don't use type assertions since this is partly transpiled by typescript
+      typeAssertions: false,
+      modules: 'commonjs'
+    }
   });
 
   // Transform all tests to make them runnable in node