feat(broccoli): improve merge-trees plugin and add "overwrite" option

tools/broccoli/broccoli-merge-trees.spec.ts

@@ -21,7 +21,7 @@ describe('MergeTrees', () => {
 
   function read(path) { return fs.readFileSync(path, "utf-8"); }
 
-  it('should copy the file from the right-most inputTree', () => {
+  it('should copy the file from the right-most inputTree with overwrite=true', () => {
     let testDir: any = {
       'tree1': {'foo.js': mockfs.file({content: 'tree1/foo.js content', mtime: new Date(1000)})},
       'tree2': {'foo.js': mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)})},
@@ -29,7 +29,7 @@ describe('MergeTrees', () => {
     };
     mockfs(testDir);
     let treeDiffer = MakeTreeDiffers(['tree1', 'tree2', 'tree3']);
-    let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {});
+    let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {overwrite: true});
     treeMerger.rebuild(treeDiffer.diffTrees());
     expect(read('dest/foo.js')).toBe('tree3/foo.js content');
 
@@ -44,4 +44,25 @@ describe('MergeTrees', () => {
     treeMerger.rebuild(treeDiffer.diffTrees());
     expect(read('dest/foo.js')).toBe('tree2/foo.js content');
   });
+
+  it('should throw if duplicates are used by default', () => {
+    let testDir: any = {
+      'tree1': {'foo.js': mockfs.file({content: 'tree1/foo.js content', mtime: new Date(1000)})},
+      'tree2': {'foo.js': mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)})},
+      'tree3': {'foo.js': mockfs.file({content: 'tree3/foo.js content', mtime: new Date(1000)})}
+    };
+    mockfs(testDir);
+    let treeDiffer = MakeTreeDiffers(['tree1', 'tree2', 'tree3']);
+    let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {});
+    expect(() => treeMerger.rebuild(treeDiffer.diffTrees())).toThrow();
+
+    delete testDir.tree2['foo.js'];
+    delete testDir.tree3['foo.js'];
+    mockfs(testDir);
+    expect(() => treeMerger.rebuild(treeDiffer.diffTrees())).not.toThrow();
+
+    testDir.tree2['foo.js'] = mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)});
+    mockfs(testDir);
+    expect(() => treeMerger.rebuild(treeDiffer.diffTrees())).toThrow();
+  });
 });

tools/broccoli/broccoli-merge-trees.ts

@@ -4,81 +4,114 @@ import path = require('path');
 var symlinkOrCopySync = require('symlink-or-copy').sync;
 import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin';
 
-function pathExists(filePath) {
+interface MergeTreesOptions {
-  try {
+  overwrite?: boolean;
-    if (fs.statSync(filePath)) {
-      return true;
-    }
-  } catch (e) {
-    if (e.code !== "ENOENT") {
-      throw e;
-    }
-  }
-  return false;
 }
 
 function outputFileSync(sourcePath, destPath) {
   let dirname = path.dirname(destPath);
   fse.mkdirsSync(dirname, {fs: fs});
-  fse.removeSync(destPath);
   symlinkOrCopySync(sourcePath, destPath);
 }
 
 export class MergeTrees implements DiffingBroccoliPlugin {
-  private mergedPaths: {[key: string]: number} = Object.create(null);
+  private pathCache: {[key: string]: number[]} = Object.create(null);
+  public options: MergeTreesOptions;
+  private firstBuild: boolean = true;
 
-  constructor(public inputPaths: string[], public cachePath: string, public options) {}
+  constructor(public inputPaths: string[], public cachePath: string,
+              options: MergeTreesOptions = {}) {
+    this.options = options || {};
+  }
 
   rebuild(treeDiffs: DiffResult[]) {
-    treeDiffs.forEach((treeDiff: DiffResult, index) => {
+    let overwrite = this.options.overwrite;
-      let inputPath = this.inputPaths[index];
+    let pathsToEmit: string[] = [];
-      let existsLater = (relativePath) => {
+    let pathsToRemove: string[] = [];
-        for (let i = treeDiffs.length - 1; i > index; --i) {
+    let emitted: {[key: string]: boolean} = Object.create(null);
-          if (pathExists(path.join(this.inputPaths[i], relativePath))) {
+    let contains = (cache, val) => {
-            return true;
+      for (let i = 0, ii = cache.length; i < ii; ++i) {
+        if (cache[i] === val) return true;
+      }
+      return false;
+    };
+
+    let emit = (relativePath) => {
+      // ASSERT(!emitted[relativePath]);
+      pathsToEmit.push(relativePath);
+      emitted[relativePath] = true;
+    };
+
+    if (this.firstBuild) {
+      // Build initial cache
+      treeDiffs.reverse().forEach((treeDiff: DiffResult, index) => {
+        index = treeDiffs.length - 1 - index;
+        treeDiff.changedPaths.forEach((changedPath) => {
+          let cache = this.pathCache[changedPath];
+          if (cache === undefined) {
+            this.pathCache[changedPath] = [index];
+            pathsToEmit.push(changedPath);
+          } else if (overwrite) {
+            // ASSERT(contains(pathsToEmit, changedPath));
+            cache.unshift(index);
+          } else {
+            throw new Error("`overwrite` option is required for handling duplicates.");
           }
-        }
+        });
-        return false;
-      };
-      let existsSooner = (relativePath) => {
-        for (let i = index - 1; i >= 0; --i) {
-          if (pathExists(path.join(this.inputPaths[i], relativePath))) {
-            return i;
-          }
-        }
-        return -1;
-      };
-      treeDiff.changedPaths.forEach((changedPath) => {
-        let inputTreeIndex = this.mergedPaths[changedPath];
-        if (inputTreeIndex !== index && !existsLater(changedPath)) {
-          inputTreeIndex = this.mergedPaths[changedPath] = index;
-          let sourcePath = path.join(inputPath, changedPath);
-          let destPath = path.join(this.cachePath, changedPath);
-          outputFileSync(sourcePath, destPath);
-        }
       });
+      this.firstBuild = false;
+    } else {
+      // Update cache
+      treeDiffs.reverse().forEach((treeDiff: DiffResult, index) => {
+        index = treeDiffs.length - 1 - index;
+        treeDiff.removedPaths.forEach((removedPath) => {
+          let cache = this.pathCache[removedPath];
+          // ASSERT(cache !== undefined);
+          // ASSERT(contains(cache, index));
+          if (cache[cache.length - 1] === index) {
+            pathsToRemove.push(path.join(this.cachePath, removedPath));
+            cache.pop();
+            if (cache.length === 0) {
+              this.pathCache[removedPath] = undefined;
+            } else if (!emitted[removedPath]) {
+              if (cache.length === 1 && !overwrite) {
+                throw new Error("`overwrite` option is required for handling duplicates.");
+              }
+              emit(removedPath);
+            }
+          }
+        });
+        treeDiff.changedPaths.forEach((changedPath) => {
+          let cache = this.pathCache[changedPath];
+          if (cache === undefined) {
+            // File was added
+            this.pathCache[changedPath] = [index];
+            emit(changedPath);
+          } else if (!contains(cache, index)) {
+            cache.push(index);
+            cache.sort((a, b) => a - b);
+            if (cache.length > 1 && !overwrite) {
+              throw new Error("`overwrite` option is required for handling duplicates.");
+            }
+            if (cache[cache.length - 1] === index && !emitted[changedPath]) {
+              emit(changedPath);
+            }
+          }
+        });
+      });
+    }
 
-      treeDiff.removedPaths.forEach((removedPath) => {
+    pathsToRemove.forEach((destPath) => fse.removeSync(destPath));
-        let inputTreeIndex = this.mergedPaths[removedPath];
+    pathsToEmit.forEach((emittedPath) => {
-
+      let cache = this.pathCache[emittedPath];
-        // if inputTreeIndex !== index, this same file was handled during
+      let destPath = path.join(this.cachePath, emittedPath);
-        // changedPaths handling
+      let sourceIndex = cache[cache.length - 1];
-        if (inputTreeIndex !== index) return;
+      let sourceInputPath = this.inputPaths[sourceIndex];
-
+      let sourcePath = path.join(sourceInputPath, emittedPath);
-        let destPath = path.join(this.cachePath, removedPath);
+      if (cache.length > 1) {
         fse.removeSync(destPath);
-        let newInputTreeIndex = existsSooner(removedPath);
+      }
-
+      outputFileSync(sourcePath, destPath);
-        // Update cached value (to either newInputTreeIndex value or undefined)
-        this.mergedPaths[removedPath] = newInputTreeIndex;
-
-        if (newInputTreeIndex >= 0) {
-          // Copy the file from the newInputTreeIndex inputPath if necessary.
-          let newInputPath = this.inputPaths[newInputTreeIndex];
-          let sourcePath = path.join(newInputPath, removedPath);
-          outputFileSync(sourcePath, destPath);
-        }
-      });
     });
   }
 }