build(broccoli): refactor typescript plugin to be incremental via DiffingBroccoliPlugin

gulpfile.js

@@ -146,7 +146,10 @@ gulp.task('build/format.dart', rundartpackage(gulp, gulpPlugins, {
 
 function doCheckFormat() {
   return gulp.src(['Brocfile*.js', 'modules/**/*.ts', 'tools/**/*.ts', '!**/typings/**/*.d.ts',
-                   '!tools/broccoli/tree-differ.ts']) // See https://github.com/angular/clang-format/issues/4
+                   // skipped  due to https://github.com/angular/clang-format/issues/4
+                   '!tools/broccoli/tree-differ.ts',
+                   // skipped  due to https://github.com/angular/gulp-clang-format/issues/3
+                   '!tools/broccoli/broccoli-typescript.ts' ])
     .pipe(format.checkFormat('file'));
 }
 

tools/broccoli/broccoli-typescript.ts

@@ -0,0 +1,172 @@
+/// <reference path="../typings/node/node.d.ts" />
+/// <reference path="../../node_modules/typescript/bin/typescript.d.ts" />
+
+import fs = require('fs');
+import fse = require('fs-extra');
+import path = require('path');
+import ts = require('typescript');
+import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin';
+
+
+type FileRegistry = ts.Map<{version: number}>;
+
+const FS_OPTS = {encoding: 'utf-8'};
+
+
+/**
+ * Broccoli plugin that implements incremental Typescript compiler.
+ *
+ * It instantiates a typescript compiler instance that keeps all the state about the project and
+ * can reemit only the files that actually changed.
+ *
+ * Limitations: only files that map directly to the changed source file via naming conventions are
+ * reemited. This primarily affects code that uses `const enum`s, because changing the enum value
+ * requires global emit, which can affect many files.
+ */
+class DiffingTSCompiler implements DiffingBroccoliPlugin {
+  private tsOpts: ts.CompilerOptions;
+  private fileRegistry: FileRegistry = Object.create(null);
+  private rootFilePaths: string[];
+  private tsServiceHost: ts.LanguageServiceHost;
+  private tsService: ts.LanguageService;
+
+
+  constructor(public inputPath: string, public cachePath: string, public options) {
+    this.tsOpts = Object.create(options);
+    this.tsOpts.outDir = this.cachePath;
+    this.tsOpts.target = (<any>ts).ScriptTarget[options.target];
+    this.rootFilePaths = options.rootFilePaths ? options.rootFilePaths.splice(0) : [];
+    this.tsServiceHost = new CustomLanguageServiceHost(this.tsOpts, this.rootFilePaths,
+                                                       this.fileRegistry, this.inputPath);
+    this.tsService = ts.createLanguageService(this.tsServiceHost, ts.createDocumentRegistry())
+  }
+
+
+  rebuild(treeDiff: DiffResult) {
+    let pathsToEmit = [];
+    let pathsWithErrors = [];
+
+    treeDiff.changedPaths.filter((changedPath) =>
+                                     changedPath.match(/\.ts/) && !changedPath.match(/\.d\.ts/))
+        .forEach((tsFilePath) => {
+          if (!this.fileRegistry[tsFilePath]) {
+            this.fileRegistry[tsFilePath] = {version: 0};
+            this.rootFilePaths.push(tsFilePath);
+          } else {
+            this.fileRegistry[tsFilePath].version++;
+          }
+
+          pathsToEmit.push(tsFilePath);
+        });
+
+    treeDiff.removedPaths.filter((changedPath) =>
+                                     changedPath.match(/\.ts/) && !changedPath.match(/\.d\.ts/))
+        .forEach((tsFilePath) => {
+          console.log('removing outputs for', tsFilePath);
+
+          this.rootFilePaths.splice(this.rootFilePaths.indexOf(tsFilePath), 1);
+          this.fileRegistry[tsFilePath] = null;
+
+          let jsFilePath = tsFilePath.replace(/\.ts$/, '.js');
+          let mapFilePath = tsFilePath.replace(/.ts$/, '.js.map');
+          let dtsFilePath = tsFilePath.replace(/\.ts$/, '.d.ts');
+
+          fs.unlinkSync(path.join(this.cachePath, jsFilePath));
+          fs.unlinkSync(path.join(this.cachePath, mapFilePath));
+          fs.unlinkSync(path.join(this.cachePath, dtsFilePath));
+        });
+
+    pathsToEmit.forEach((tsFilePath) => {
+      let output = this.tsService.getEmitOutput(tsFilePath);
+
+      if (output.emitSkipped) {
+        let errorFound = this.logError(tsFilePath);
+        if (errorFound) {
+          pathsWithErrors.push(tsFilePath);
+        }
+      } else {
+        output.outputFiles.forEach(o => {
+          let destDirPath = path.dirname(o.name);
+          fse.mkdirsSync(destDirPath);
+          fs.writeFileSync(o.name, o.text, FS_OPTS);
+        });
+      }
+    });
+
+    if (pathsWithErrors.length) {
+      throw new Error('Typescript found errors listed above...');
+    }
+  }
+
+
+  private logError(tsFilePath) {
+    let allDiagnostics = this.tsService.getCompilerOptionsDiagnostics()
+                             .concat(this.tsService.getSyntacticDiagnostics(tsFilePath))
+                             .concat(this.tsService.getSemanticDiagnostics(tsFilePath));
+
+    allDiagnostics.forEach(diagnostic => {
+      let message = ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n");
+      if (diagnostic.file) {
+        let{line, character} = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
+        console.log(`  Error ${diagnostic.file.fileName} (${line + 1},${character + 1}): ` +
+                    `${message}`);
+      } else {
+        console.log(`  Error: ${message}`);
+      }
+    });
+
+    return !!allDiagnostics.length;
+  }
+}
+
+
+class CustomLanguageServiceHost implements ts.LanguageServiceHost {
+  private currentDirectory: string;
+  private defaultLibFilePath: string;
+
+
+  constructor(private compilerOptions: ts.CompilerOptions, private fileNames: string[],
+              private fileRegistry: FileRegistry, private treeInputPath: string) {
+    this.currentDirectory = process.cwd();
+    this.defaultLibFilePath = ts.getDefaultLibFilePath(compilerOptions);
+  }
+
+
+  getScriptFileNames(): string[] { return this.fileNames; }
+
+
+  getScriptVersion(fileName: string): string {
+    return this.fileRegistry[fileName] && this.fileRegistry[fileName].version.toString();
+  }
+
+
+  getScriptSnapshot(tsFilePath: string): ts.IScriptSnapshot {
+    // TODO: this method is called a lot, add cache
+
+    let absoluteTsFilePath = (tsFilePath == this.defaultLibFilePath) ?
+                                 tsFilePath :
+                                 path.join(this.treeInputPath, tsFilePath);
+
+    if (!fs.existsSync(absoluteTsFilePath)) {
+      // TypeScript seems to request lots of bogus paths during import path lookup and resolution,
+      // so we we just return undefined when the path is not correct.
+      return undefined;
+    }
+    return ts.ScriptSnapshot.fromString(fs.readFileSync(absoluteTsFilePath, FS_OPTS));
+  }
+
+
+  getCurrentDirectory(): string { return this.currentDirectory; }
+
+
+  getCompilationSettings(): ts.CompilerOptions { return this.compilerOptions; }
+
+
+  getDefaultLibFileName(options: ts.CompilerOptions): string {
+    // ignore options argument, options should not change during the lifetime of the plugin
+    return this.defaultLibFilePath;
+  }
+}
+
+
+export default wrapDiffingPlugin(DiffingTSCompiler);

tools/broccoli/diffing-broccoli-plugin.ts

@@ -62,6 +62,7 @@ class DiffingPluginWrapper implements BroccoliTree {
 
 
   rebuild() {
+    try {
       let firstRun = !this.initialized;
       this.init();
 
@@ -75,6 +76,10 @@ class DiffingPluginWrapper implements BroccoliTree {
       }
 
       this.relinkOutputAndCachePaths();
+    } catch (e) {
+      e.message = `[${this.description}]: ${e.message}`;
+      throw e;
+    }
   }
 
 

tools/broccoli/trees/browser_tree.ts

@@ -8,8 +8,8 @@ var path = require('path');
 var replace = require('broccoli-replace');
 var stew = require('broccoli-stew');
 var ts2dart = require('../broccoli-ts2dart');
-var TypescriptCompiler = require('../typescript');
 
+import compileWithTypescript from '../broccoli-typescript';
 import destCopy from '../broccoli-dest-copy';
 import {default as transpileWithTraceur, TRACEUR_RUNTIME_PATH} from '../traceur/index';
 
@@ -42,20 +42,26 @@ module.exports = function makeBrowserTree(options, destinationPath) {
   // Use TypeScript to transpile the *.ts files to ES6
   // We don't care about errors: we let the TypeScript compilation to ES5
   // in node_tree.ts do the type-checking.
-  var typescriptTree = new TypescriptCompiler(modulesTree, {
+  var typescriptTree = compileWithTypescript(modulesTree, {
-    target: 'ES6',
-    sourceMap: true,
-    mapRoot: '', /* force sourcemaps to use relative path */
     allowNonTsExtensions: false,
-    typescript: require('typescript'),
+    declaration: true,
-    noEmitOnError: false,
     emitDecoratorMetadata: true,
-    outDir: 'angular2'
+    mapRoot: '',           // force sourcemaps to use relative path
+    noEmitOnError: false,  // temporarily ignore errors, we type-check only via cjs build
+    rootDir: '.',
+    sourceMap: true,
+    sourceRoot: '.',
+    target: 'ES6'
   });
   typescriptTree = stew.rename(typescriptTree, '.js', '.es6');
 
   var es6Tree = mergeTrees([traceurTree, typescriptTree]);
 
+  // TODO(iminar): tree differ seems to have issues with trees created by mergeTrees, investigate!
+  //   ENOENT error is thrown while doing fs.readdirSync on inputRoot
+  //   in the meantime, we just do noop mv to create a new tree
+  es6Tree = stew.mv(es6Tree, '');
+
   // Call Traceur again to lower the ES6 build tree to ES5
   var es5Tree = transpileWithTraceur(es6Tree, {
     destExtension: '.js',

tools/broccoli/trees/node_tree.ts

@@ -9,7 +9,7 @@ var replace = require('broccoli-replace');
 var stew = require('broccoli-stew');
 var ts2dart = require('../broccoli-ts2dart');
 import transpileWithTraceur from '../traceur/index';
-var TypescriptCompiler = require('../typescript');
+import compileWithTypescript from '../broccoli-typescript';
 
 var projectRootDir = path.normalize(path.join(__dirname, '..', '..', '..', '..'));
 
@@ -98,17 +98,20 @@ module.exports = function makeNodeTree(destinationPath) {
       packageJsons, {files: ["**/**"], context: {'packageJson': COMMON_PACKAGE_JSON}});
 
 
-  var typescriptTree = new TypescriptCompiler(modulesTree, {
+  var typescriptTree = compileWithTypescript(modulesTree, {
-    target: 'ES5',
+    allowNonTsExtensions: false,
-    sourceMap: true,
+    declaration: true,
     mapRoot: '', /* force sourcemaps to use relative path */
     module: 'commonjs',
-    allowNonTsExtensions: false,
-    typescript: require('typescript'),
     noEmitOnError: true,
-    outDir: 'angular2'
+    rootDir: '.',
+    rootFilePaths: ['angular2/traceur-runtime.d.ts'],
+    sourceMap: true,
+    sourceRoot: '.',
+    target: 'ES5'
   });
 
+
   nodeTree = mergeTrees([nodeTree, typescriptTree, docs, packageJsons]);
 
   // TODO(iminar): tree differ seems to have issues with trees created by mergeTrees, investigate!

tools/broccoli/typescript/index.ts

@@ -1,55 +0,0 @@
-/// <reference path="../broccoli-writer.d.ts" />
-/// <reference path="../../typings/node/node.d.ts" />
-/// <reference path="../../../node_modules/typescript/bin/typescript.d.ts" />
-
-import fs = require('fs');
-import path = require('path');
-import ts = require('typescript');
-var walkSync = require('walk-sync');
-import Writer = require('broccoli-writer');
-var xtend = require('xtend');
-
-class TSCompiler extends Writer {
-  constructor(private inputTree, private options: ts.CompilerOptions = {}) { super(); }
-
-  write(readTree, destDir) {
-    var options: ts.CompilerOptions = xtend({outDir: destDir}, this.options);
-    if (this.options.outDir) {
-      options.outDir = path.resolve(destDir, options.outDir);
-    }
-    if (options.out) {
-      options.out = path.resolve(destDir, options.out);
-    }
-    options.target = (<any>ts).ScriptTarget[options.target];
-    return readTree(this.inputTree)
-        .then(srcDir => {
-          var files = walkSync(srcDir)
-                          .filter(filepath => path.extname(filepath).toLowerCase() === '.ts')
-                          .map(filepath => path.resolve(srcDir, filepath));
-
-          if (files.length > 0) {
-            var program = ts.createProgram(files, options);
-            var emitResult = program.emit();
-
-            var allDiagnostics = ts.getPreEmitDiagnostics(program).concat(emitResult.diagnostics);
-
-            var errMsg = '';
-            allDiagnostics.forEach(diagnostic => {
-              var message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n');
-              if (!diagnostic.file) {
-                errMsg += `\n${message}`;
-                return;
-              }
-              var {line, character} =
-                  diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
-              errMsg += `\n${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`;
-            });
-
-            if (emitResult.emitSkipped) {
-              throw new Error(errMsg);
-            }
-          }
-        });
-  }
-}
-module.exports = TSCompiler;