build(broccoli): refactor typescript plugin to be incremental via DiffingBroccoliPlugin

This commit is contained in:
Igor Minar 2015-05-04 08:27:14 -07:00
parent 9d1df21d91
commit 1d0078415f
6 changed files with 214 additions and 80 deletions

View File

@ -146,7 +146,10 @@ gulp.task('build/format.dart', rundartpackage(gulp, gulpPlugins, {
function doCheckFormat() { function doCheckFormat() {
return gulp.src(['Brocfile*.js', 'modules/**/*.ts', 'tools/**/*.ts', '!**/typings/**/*.d.ts', return gulp.src(['Brocfile*.js', 'modules/**/*.ts', 'tools/**/*.ts', '!**/typings/**/*.d.ts',
'!tools/broccoli/tree-differ.ts']) // See https://github.com/angular/clang-format/issues/4 // skipped due to https://github.com/angular/clang-format/issues/4
'!tools/broccoli/tree-differ.ts',
// skipped due to https://github.com/angular/gulp-clang-format/issues/3
'!tools/broccoli/broccoli-typescript.ts' ])
.pipe(format.checkFormat('file')); .pipe(format.checkFormat('file'));
} }

View File

@ -0,0 +1,172 @@
/// <reference path="../typings/node/node.d.ts" />
/// <reference path="../../node_modules/typescript/bin/typescript.d.ts" />
import fs = require('fs');
import fse = require('fs-extra');
import path = require('path');
import ts = require('typescript');
import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin';
type FileRegistry = ts.Map<{version: number}>;
const FS_OPTS = {encoding: 'utf-8'};
/**
* Broccoli plugin that implements incremental Typescript compiler.
*
* It instantiates a typescript compiler instance that keeps all the state about the project and
* can reemit only the files that actually changed.
*
* Limitations: only files that map directly to the changed source file via naming conventions are
* reemited. This primarily affects code that uses `const enum`s, because changing the enum value
* requires global emit, which can affect many files.
*/
class DiffingTSCompiler implements DiffingBroccoliPlugin {
private tsOpts: ts.CompilerOptions;
private fileRegistry: FileRegistry = Object.create(null);
private rootFilePaths: string[];
private tsServiceHost: ts.LanguageServiceHost;
private tsService: ts.LanguageService;
constructor(public inputPath: string, public cachePath: string, public options) {
this.tsOpts = Object.create(options);
this.tsOpts.outDir = this.cachePath;
this.tsOpts.target = (<any>ts).ScriptTarget[options.target];
this.rootFilePaths = options.rootFilePaths ? options.rootFilePaths.splice(0) : [];
this.tsServiceHost = new CustomLanguageServiceHost(this.tsOpts, this.rootFilePaths,
this.fileRegistry, this.inputPath);
this.tsService = ts.createLanguageService(this.tsServiceHost, ts.createDocumentRegistry())
}
rebuild(treeDiff: DiffResult) {
let pathsToEmit = [];
let pathsWithErrors = [];
treeDiff.changedPaths.filter((changedPath) =>
changedPath.match(/\.ts/) && !changedPath.match(/\.d\.ts/))
.forEach((tsFilePath) => {
if (!this.fileRegistry[tsFilePath]) {
this.fileRegistry[tsFilePath] = {version: 0};
this.rootFilePaths.push(tsFilePath);
} else {
this.fileRegistry[tsFilePath].version++;
}
pathsToEmit.push(tsFilePath);
});
treeDiff.removedPaths.filter((changedPath) =>
changedPath.match(/\.ts/) && !changedPath.match(/\.d\.ts/))
.forEach((tsFilePath) => {
console.log('removing outputs for', tsFilePath);
this.rootFilePaths.splice(this.rootFilePaths.indexOf(tsFilePath), 1);
this.fileRegistry[tsFilePath] = null;
let jsFilePath = tsFilePath.replace(/\.ts$/, '.js');
let mapFilePath = tsFilePath.replace(/.ts$/, '.js.map');
let dtsFilePath = tsFilePath.replace(/\.ts$/, '.d.ts');
fs.unlinkSync(path.join(this.cachePath, jsFilePath));
fs.unlinkSync(path.join(this.cachePath, mapFilePath));
fs.unlinkSync(path.join(this.cachePath, dtsFilePath));
});
pathsToEmit.forEach((tsFilePath) => {
let output = this.tsService.getEmitOutput(tsFilePath);
if (output.emitSkipped) {
let errorFound = this.logError(tsFilePath);
if (errorFound) {
pathsWithErrors.push(tsFilePath);
}
} else {
output.outputFiles.forEach(o => {
let destDirPath = path.dirname(o.name);
fse.mkdirsSync(destDirPath);
fs.writeFileSync(o.name, o.text, FS_OPTS);
});
}
});
if (pathsWithErrors.length) {
throw new Error('Typescript found errors listed above...');
}
}
private logError(tsFilePath) {
let allDiagnostics = this.tsService.getCompilerOptionsDiagnostics()
.concat(this.tsService.getSyntacticDiagnostics(tsFilePath))
.concat(this.tsService.getSemanticDiagnostics(tsFilePath));
allDiagnostics.forEach(diagnostic => {
let message = ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n");
if (diagnostic.file) {
let{line, character} = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
console.log(` Error ${diagnostic.file.fileName} (${line + 1},${character + 1}): ` +
`${message}`);
} else {
console.log(` Error: ${message}`);
}
});
return !!allDiagnostics.length;
}
}
class CustomLanguageServiceHost implements ts.LanguageServiceHost {
private currentDirectory: string;
private defaultLibFilePath: string;
constructor(private compilerOptions: ts.CompilerOptions, private fileNames: string[],
private fileRegistry: FileRegistry, private treeInputPath: string) {
this.currentDirectory = process.cwd();
this.defaultLibFilePath = ts.getDefaultLibFilePath(compilerOptions);
}
getScriptFileNames(): string[] { return this.fileNames; }
getScriptVersion(fileName: string): string {
return this.fileRegistry[fileName] && this.fileRegistry[fileName].version.toString();
}
getScriptSnapshot(tsFilePath: string): ts.IScriptSnapshot {
// TODO: this method is called a lot, add cache
let absoluteTsFilePath = (tsFilePath == this.defaultLibFilePath) ?
tsFilePath :
path.join(this.treeInputPath, tsFilePath);
if (!fs.existsSync(absoluteTsFilePath)) {
// TypeScript seems to request lots of bogus paths during import path lookup and resolution,
// so we we just return undefined when the path is not correct.
return undefined;
}
return ts.ScriptSnapshot.fromString(fs.readFileSync(absoluteTsFilePath, FS_OPTS));
}
getCurrentDirectory(): string { return this.currentDirectory; }
getCompilationSettings(): ts.CompilerOptions { return this.compilerOptions; }
getDefaultLibFileName(options: ts.CompilerOptions): string {
// ignore options argument, options should not change during the lifetime of the plugin
return this.defaultLibFilePath;
}
}
export default wrapDiffingPlugin(DiffingTSCompiler);

View File

@ -62,19 +62,24 @@ class DiffingPluginWrapper implements BroccoliTree {
rebuild() { rebuild() {
let firstRun = !this.initialized; try {
this.init(); let firstRun = !this.initialized;
this.init();
let diffResult = this.treeDiffer.diffTree(); let diffResult = this.treeDiffer.diffTree();
diffResult.log(!firstRun); diffResult.log(!firstRun);
var rebuildPromise = this.wrappedPlugin.rebuild(diffResult); var rebuildPromise = this.wrappedPlugin.rebuild(diffResult);
if (rebuildPromise) { if (rebuildPromise) {
return (<Promise<any>>rebuildPromise).then(this.relinkOutputAndCachePaths.bind(this)); return (<Promise<any>>rebuildPromise).then(this.relinkOutputAndCachePaths.bind(this));
}
this.relinkOutputAndCachePaths();
} catch (e) {
e.message = `[${this.description}]: ${e.message}`;
throw e;
} }
this.relinkOutputAndCachePaths();
} }

View File

@ -8,8 +8,8 @@ var path = require('path');
var replace = require('broccoli-replace'); var replace = require('broccoli-replace');
var stew = require('broccoli-stew'); var stew = require('broccoli-stew');
var ts2dart = require('../broccoli-ts2dart'); var ts2dart = require('../broccoli-ts2dart');
var TypescriptCompiler = require('../typescript');
import compileWithTypescript from '../broccoli-typescript';
import destCopy from '../broccoli-dest-copy'; import destCopy from '../broccoli-dest-copy';
import {default as transpileWithTraceur, TRACEUR_RUNTIME_PATH} from '../traceur/index'; import {default as transpileWithTraceur, TRACEUR_RUNTIME_PATH} from '../traceur/index';
@ -42,20 +42,26 @@ module.exports = function makeBrowserTree(options, destinationPath) {
// Use TypeScript to transpile the *.ts files to ES6 // Use TypeScript to transpile the *.ts files to ES6
// We don't care about errors: we let the TypeScript compilation to ES5 // We don't care about errors: we let the TypeScript compilation to ES5
// in node_tree.ts do the type-checking. // in node_tree.ts do the type-checking.
var typescriptTree = new TypescriptCompiler(modulesTree, { var typescriptTree = compileWithTypescript(modulesTree, {
target: 'ES6',
sourceMap: true,
mapRoot: '', /* force sourcemaps to use relative path */
allowNonTsExtensions: false, allowNonTsExtensions: false,
typescript: require('typescript'), declaration: true,
noEmitOnError: false,
emitDecoratorMetadata: true, emitDecoratorMetadata: true,
outDir: 'angular2' mapRoot: '', // force sourcemaps to use relative path
noEmitOnError: false, // temporarily ignore errors, we type-check only via cjs build
rootDir: '.',
sourceMap: true,
sourceRoot: '.',
target: 'ES6'
}); });
typescriptTree = stew.rename(typescriptTree, '.js', '.es6'); typescriptTree = stew.rename(typescriptTree, '.js', '.es6');
var es6Tree = mergeTrees([traceurTree, typescriptTree]); var es6Tree = mergeTrees([traceurTree, typescriptTree]);
// TODO(iminar): tree differ seems to have issues with trees created by mergeTrees, investigate!
// ENOENT error is thrown while doing fs.readdirSync on inputRoot
// in the meantime, we just do noop mv to create a new tree
es6Tree = stew.mv(es6Tree, '');
// Call Traceur again to lower the ES6 build tree to ES5 // Call Traceur again to lower the ES6 build tree to ES5
var es5Tree = transpileWithTraceur(es6Tree, { var es5Tree = transpileWithTraceur(es6Tree, {
destExtension: '.js', destExtension: '.js',

View File

@ -9,7 +9,7 @@ var replace = require('broccoli-replace');
var stew = require('broccoli-stew'); var stew = require('broccoli-stew');
var ts2dart = require('../broccoli-ts2dart'); var ts2dart = require('../broccoli-ts2dart');
import transpileWithTraceur from '../traceur/index'; import transpileWithTraceur from '../traceur/index';
var TypescriptCompiler = require('../typescript'); import compileWithTypescript from '../broccoli-typescript';
var projectRootDir = path.normalize(path.join(__dirname, '..', '..', '..', '..')); var projectRootDir = path.normalize(path.join(__dirname, '..', '..', '..', '..'));
@ -98,17 +98,20 @@ module.exports = function makeNodeTree(destinationPath) {
packageJsons, {files: ["**/**"], context: {'packageJson': COMMON_PACKAGE_JSON}}); packageJsons, {files: ["**/**"], context: {'packageJson': COMMON_PACKAGE_JSON}});
var typescriptTree = new TypescriptCompiler(modulesTree, { var typescriptTree = compileWithTypescript(modulesTree, {
target: 'ES5', allowNonTsExtensions: false,
sourceMap: true, declaration: true,
mapRoot: '', /* force sourcemaps to use relative path */ mapRoot: '', /* force sourcemaps to use relative path */
module: 'commonjs', module: 'commonjs',
allowNonTsExtensions: false,
typescript: require('typescript'),
noEmitOnError: true, noEmitOnError: true,
outDir: 'angular2' rootDir: '.',
rootFilePaths: ['angular2/traceur-runtime.d.ts'],
sourceMap: true,
sourceRoot: '.',
target: 'ES5'
}); });
nodeTree = mergeTrees([nodeTree, typescriptTree, docs, packageJsons]); nodeTree = mergeTrees([nodeTree, typescriptTree, docs, packageJsons]);
// TODO(iminar): tree differ seems to have issues with trees created by mergeTrees, investigate! // TODO(iminar): tree differ seems to have issues with trees created by mergeTrees, investigate!

View File

@ -1,55 +0,0 @@
/// <reference path="../broccoli-writer.d.ts" />
/// <reference path="../../typings/node/node.d.ts" />
/// <reference path="../../../node_modules/typescript/bin/typescript.d.ts" />
import fs = require('fs');
import path = require('path');
import ts = require('typescript');
var walkSync = require('walk-sync');
import Writer = require('broccoli-writer');
var xtend = require('xtend');
class TSCompiler extends Writer {
constructor(private inputTree, private options: ts.CompilerOptions = {}) { super(); }
write(readTree, destDir) {
var options: ts.CompilerOptions = xtend({outDir: destDir}, this.options);
if (this.options.outDir) {
options.outDir = path.resolve(destDir, options.outDir);
}
if (options.out) {
options.out = path.resolve(destDir, options.out);
}
options.target = (<any>ts).ScriptTarget[options.target];
return readTree(this.inputTree)
.then(srcDir => {
var files = walkSync(srcDir)
.filter(filepath => path.extname(filepath).toLowerCase() === '.ts')
.map(filepath => path.resolve(srcDir, filepath));
if (files.length > 0) {
var program = ts.createProgram(files, options);
var emitResult = program.emit();
var allDiagnostics = ts.getPreEmitDiagnostics(program).concat(emitResult.diagnostics);
var errMsg = '';
allDiagnostics.forEach(diagnostic => {
var message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n');
if (!diagnostic.file) {
errMsg += `\n${message}`;
return;
}
var {line, character} =
diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
errMsg += `\n${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`;
});
if (emitResult.emitSkipped) {
throw new Error(errMsg);
}
}
});
}
}
module.exports = TSCompiler;