chore(devguide-tooling): node_modules symlink support for _examples and subdirs

closes #450.  READ COMMENTS IN PR450 FOR INSTRUCTIONS ON USAGE
We'll put these instructions some place better soon.

Also replace globule with globby ( perf issues) and explicitly exclude node_modules where possible.
Updated _examples/package.json for latest 3rd party libs (e.g. alpha.48) and
updated template-syntax/ts/package.json to be a prototypical example

gulpfile.js

@@ -16,6 +16,7 @@ var fs = fsExtra;
 var exec = require('child_process').exec;
 var execPromise = Q.denodeify(exec);
 var prompt = require('prompt');
+var globby = require("globby");
 
 // TODO:
 //  1. Think about using runSequence
@@ -33,6 +34,7 @@ var LIVE_EXAMPLES_PATH = path.join(RESOURCES_PATH, 'live-examples');
 var docShredder = require(path.resolve(TOOLS_PATH, 'doc-shredder/doc-shredder'));
 var exampleZipper = require(path.resolve(TOOLS_PATH, '_example-zipper/exampleZipper'));
 var plunkerBuilder = require(path.resolve(TOOLS_PATH, 'plunker-builder/plunkerBuilder'));
+var fsUtils = require(path.resolve(TOOLS_PATH, 'fs-utils/fsUtils'));
 
 var _devguideShredOptions =  {
   examplesDir: path.join(DOCS_PATH, '_examples'),
@@ -53,6 +55,8 @@ var _excludeMatchers = _excludePatterns.map(function(excludePattern){
 });
 
 
+
+
 // Public tasks
 
 gulp.task('default', ['help']);
@@ -65,6 +69,23 @@ gulp.task('help', taskListing.withFilters(function(taskName) {
   return shouldRemove;
 }));
 
+gulp.task('add-example-symlinks', function() {
+  var realPath = path.join(EXAMPLES_PATH, '/node_modules');
+  var nodeModulesPaths = getNodeModulesPaths(EXAMPLES_PATH);
+
+  nodeModulesPaths.forEach(function(linkPath) {
+    gutil.log("symlinking " + linkPath + ' -> ' + realPath)
+    fsUtils.addSymlink(realPath, linkPath);
+  });
+});
+
+gulp.task('remove-example-symlinks', function() {
+  var nodeModulesPaths = getNodeModulesPaths(EXAMPLES_PATH);
+  nodeModulesPaths.forEach(function(linkPath) {
+    fsUtils.removeSymlink(linkPath);
+  });
+});
+
 gulp.task('serve-and-sync', ['build-docs'], function (cb) {
   watchAndSync({devGuide: true, apiDocs: true, apiExamples: true, localFiles: true}, cb);
 });
@@ -147,9 +168,11 @@ gulp.task('git-changed-examples', ['_shred-devguide-examples'], function(){
   });
 });
 
-gulp.task('check-deploy', ['build-docs'], function() {
+// gulp.task('check-deploy', ['build-docs'], function() {
+gulp.task('check-deploy', function() {
   gutil.log('running harp compile...');
   return execPromise('npm run harp -- compile . ./www', {}).then(function() {
+    gutil.log('compile ok - running live server ...');
     execPromise('npm run live-server ./www');
     return askDeploy();
   }).then(function(shouldDeploy) {
@@ -161,6 +184,8 @@ gulp.task('check-deploy', ['build-docs'], function() {
     }
   }).then(function(s) {
     gutil.log(s.join(''));
+  }).catch(function(e) {
+    gutil.log(e);
   });
 });
 
@@ -172,6 +197,8 @@ gulp.task('test-api-builder', function (cb) {
 
 // Internal tasks
 
+
+
 gulp.task('_shred-devguide-examples', ['_shred-clean-devguide'], function() {
   return docShredder.shred( _devguideShredOptions);
 });
@@ -199,6 +226,19 @@ gulp.task('_zip-examples', function() {
 
 // Helper functions
 
+function getNodeModulesPaths(basePath) {
+  var jsonPattern = path.join(basePath, "**/package.json");
+  var exceptJsonPattern = "!" + path.join(basePath, "/package.json");
+  var nmPattern =  path.join(basePath, "**/node_modules/**");
+  var fileNames = globby.sync( [ jsonPattern, exceptJsonPattern ], { ignore: [nmPattern] } );
+  // same as above but perf can differ.
+  // var fileNames = globby.sync( [jsonPattern, "!" + nmPattern]);
+  var paths = fileNames.map(function(fileName) {
+    return path.join(path.dirname(fileName), "/node_modules");
+  });
+  return paths;
+}
+
 function watchAndSync(options, cb) {
 
   execCommands(['npm run harp -- server .'], {}, cb);
@@ -451,6 +491,7 @@ function addKeyValue(map, key, value) {
   }
 }
 
+
 // Synchronously execute a chain of commands.
 // cmds: an array of commands
 // options: { shouldLog: true,  shouldThrow: true }

package.json

@@ -34,7 +34,7 @@
     "diff": "^2.1.3",
     "fs-extra": "^0.24.0",
     "glob": "^5.0.14",
-    "globule": "^0.2.0",
+    "globby": "^4.0.0",
     "gulp": "^3.5.6",
     "gulp-task-listing": "^1.0.1",
     "gulp-util": "^3.0.6",

public/docs/_examples/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "angular2-examples-shared",
+  "version": "1.0.0",
+  "description": "This package.json file is intended to be the superset of all dependencies for all of the package.json files below this point.",
+  "main": "index.js",
+  "scripts": {
+    "tsc": "tsc -p src -w",
+    "start": "live-server --open=src",
+    "both": "concurrent \"npm run tsc\" \"npm run start\" "
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "angular2": "2.0.0-alpha.48",
+    "systemjs": "0.19.6"
+  },
+  "devDependencies": {
+    "concurrently": "^1.0.0",
+    "live-server": "^0.8.2",
+    "lite-server": "^1.3.1",   
+    "typescript": "^1.7.3"
+  }
+}

public/docs/_examples/template-syntax/ts/package.json

@@ -5,17 +5,10 @@
   "main": "index.js",
   "scripts": {
     "tsc": "tsc -p src -w",
-    "start": "live-server --open=src"
+    "start": "live-server --open=src",
+    "both": "concurrent \"npm run tsc\" \"npm run start\" "
   },
   "keywords": [],
   "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "angular2": "2.0.0-alpha.48",
-    "systemjs": "0.19.6"
-  },
-  "devDependencies": {
-    "live-server": "^0.8.1",
-    "typescript": "^1.7.2"
-  }
+  "license": "ISC"
 }

tools/_example-zipper/exampleZipper.js

@@ -10,8 +10,7 @@ var archiver = require('archiver');
 
 var fs = require('fs');
 var mkdirp = require('mkdirp');
-
-var globule = require('globule');
+var globby = require('globby');
 
 var regionExtractor = require('../doc-shredder/regionExtractor');
 
@@ -19,7 +18,7 @@ var regionExtractor = require('../doc-shredder/regionExtractor');
 // globs is an array of globs
 function zipExamples(sourceDirName, outputDirName) {
   var gpath = path.join(sourceDirName, '**/*zipconfig.json');
-  var configFileNames = globule.find([gpath]);
+  var configFileNames = globby.sync([gpath], { ignore: ['**/node_modules/**'] });
   configFileNames.forEach(function(configFileName) {
     zipExample(configFileName, sourceDirName, outputDirName);
   });
@@ -46,7 +45,12 @@ function zipExample(configFileName, sourceDirName, outputDirName) {
   }
 
   var outputFileName = path.join(outputDirName, relativeDirName, baseFileName + ".zip");
-  var fileNames = globule.find(fileGlobs, { srcBase: configDirName, prefixBase: configDirName });
+  // old code
+  // var fileNames = globule.find(fileGlobs, { srcBase: configDirName, prefixBase: configDirName });
+  fileGlobs = fileGlobs.map(function(fileGlob) {
+    return path.join(configDirName, fileGlob);
+  });
+  var fileNames = globby.sync(fileGlobs, { ignore: ["**/node_modules/**"] });
 
   var zip = createZipArchive(outputFileName);
   fileNames.forEach(function(fileName) {

tools/doc-shredder/_test/doc-shredder.spec.js

@@ -3,7 +3,7 @@ var Q = require("q");
 var del = require('del')
 // delPromise is a 'promise' version of del
 var delPromise =  Q.denodeify(del);
-var globule = require('globule');
+var globby = require('globby');
 var path = require('canonical-path');
 var fs = require('fs');
 var JsDiff = require('diff');
@@ -25,7 +25,7 @@ describe('doc-shredder', function() {
       return shred(_shredOptions);
     }).then(function() {
       var mdOvrPath = path.join(_shredOptions.fragmentsDir, '**/*.ovr.*');
-      var fileNames = globule.find([mdOvrPath]);
+      var fileNames = globby.sync([mdOvrPath], { ignore: ["**/node_modules/**"] });
       var errs = [];
       fileNames.forEach(function(fileName) {
         console.log('comparing: ' + fileName);

tools/fs-utils/fsUtils.js

@@ -0,0 +1,137 @@
+var path = require('canonical-path');
+var fs = require('fs');
+var minimatch = require("minimatch");
+
+
+// Utility wrappers over fs functions
+
+module.exports = {
+  addSymlink: addSymlink,
+  removeSymlink: removeSymlink,
+  removeDirSync: removeDirSync,
+  lstatSyncExt: lstatSyncExt,
+};
+
+// create a new link (linkPath) that points
+// to realPath - will also delete the linkPath
+// if it is a preexisting folder and not itself a symbolic link before it
+// starts.
+function addSymlink(realPath, linkPath) {
+
+  // check if the linkPath is a folder
+  // and if so delete it
+  var stat = lstatSyncExt(linkPath);
+
+  if (stat && stat.isSymbolicLink()) {
+    // todo: check if it points to realPath
+    return;
+  }
+  if (stat && stat.isDirectory()) {
+    removeDirSync(linkPath);
+  }
+  symLinkSyncExt(realPath, linkPath, 'dir');
+
+}
+
+function removeSymlink(linkPath) {
+  var stat = lstatSyncExt(linkPath);
+  if (!stat || !stat.isSymbolicLink()) return;
+
+  fs.unlinkSync(linkPath);
+}
+
+// remove a dir and all of its files - sync
+function removeDirSync(dirPath) {
+  var files;
+  try {
+    files = fs.readdirSync(dirPath);
+  }
+  catch(e) { return; }
+  if (files.length > 0) {
+    for (var i = 0; i < files.length; i++) {
+      var filePath = path.join(dirPath, files[i]);
+      var stat = fs.lstatSync(filePath);
+      if (stat.isFile() || stat.isSymbolicLink()) {
+        fs.unlinkSync(filePath);
+      } else {
+        removeDirSync(filePath);
+      }
+    }
+  }
+  fs.rmdirSync(dirPath);
+}
+
+// same as lstatSync except returns a null for nonExistent file instead of throwing
+function lstatSyncExt(filePath) {
+  try {
+    var stat = fs.lstatSync(filePath);
+  } catch (e) {
+    if(e.code == 'ENOENT') return null;
+    throw e;
+  }
+  return stat;
+}
+
+function symLinkSyncExt(realPath, linkPath, type) {
+  // fs.symlink requires the real path to be fully resolved.
+  realPath = path.resolve(realPath);
+  try {
+    fs.symlinkSync(realPath, linkPath, type);
+  } catch (e) {
+    var msg = "Unable to create symlink: " + linkPath;
+    if (e.code == "EPERM") {
+      msg += "\nPermissions issue, on windows this function must be run as an administrator."
+    }
+    console.log(msg + "\n" + e);
+  }
+}
+
+// Talk to Jay - not currently needed.
+//
+//function globWithIgnore(basePath, globs, ignoreGlobs, options) {
+//  options = options || {};
+//  options.filters = globs.map(function (glob) {
+//    return minimatch.filter(glob);
+//  });
+//  options.ignoreFilters = ignoreGlobs && ignoreGlobs.map(function(glob) {
+//      return minimatch.filter(glob);
+//    });
+//  options.ignoreSymlink = options.ignoreSymlink == null ? true : false;
+//  return globSyncCore(basePath, options);
+//}
+//
+//function globSyncCore(basePath, options) {
+//  var results = [];
+//  var files;
+//  try {
+//    files = fs.readdirSync(basePath);
+//  }
+//  catch(e) { return results; }
+//
+//  files.forEach(function(file) {
+//    var filePath = path.join(basePath, file);
+//    var shouldIgnore = options.ignoreFilters && options.ignoreFilters.some(function(filter) {
+//        return filter(filePath);
+//      });
+//    if (shouldIgnore) return;
+//
+//    var stat = fs.lstatSync(filePath);
+//
+//    if (stat.isDirectory()) {
+//      if (stat.isSymbolicLink() && options.ignoreSymlink) {
+//        // do nothing
+//      } else {
+//        var partialResults = globSyncCore(filePath, options);
+//        Array.prototype.push.apply(results, partialResults);
+//      }
+//    } else {
+//      var ok = options.filters.every(function(filter) {
+//        return filter(filePath);
+//      });
+//      if (ok) {
+//        results.push(filePath);
+//      }
+//    }
+//  });
+//  return results;
+//}

tools/plunker-builder/plunkerBuilder.js

@@ -4,7 +4,7 @@ var Q = require('q');
 var _ = require('lodash');
 var jsdom = require("jsdom");
 var fs = require("fs");
-var globule = require('globule');
+var globby = require('globby');
 var mkdirp = require('mkdirp');
 
 var indexHtmlTranslator = require('./indexHtmlTranslator');
@@ -20,7 +20,7 @@ function buildPlunkers(basePath, destPath, options) {
   var gpaths = configExtns.map(function(extn) {
     return path.join(basePath, '**/' + extn);
   });
-  var fileNames = globule.find(gpaths);
+  var fileNames = globby.sync(gpaths, { ignore: "**/node_modules/**"});
   fileNames.forEach(function(configFileName) {
     try {
       buildPlunkerFrom(configFileName, basePath, destPath);
@@ -94,10 +94,11 @@ function initConfigAndCollectFileNames(configFileName) {
       return path.join(basePath, fileName);
     }
   });
-  var defaultExcludes = [ '!**/node_modules/**','!**/typings/**','!**/tsconfig.json', '!**/*plnkr.json', '!**/*plnkr.html', '!**/*plnkr.no-link.html' ];
+  // var defaultExcludes = [ '!**/node_modules/**','!**/typings/**','!**/tsconfig.json', '!**/*plnkr.json', '!**/*plnkr.html', '!**/*plnkr.no-link.html' ];
+  var defaultExcludes = [ '!**/typings/**','!**/tsconfig.json', '!**/*plnkr.json', '!**/*plnkr.html', '!**/*plnkr.no-link.html' ];
   Array.prototype.push.apply(gpaths, defaultExcludes);
 
-  config.fileNames = globule.find(gpaths);
+  config.fileNames = globby.sync(gpaths, { ignore: ["**/node_modules/**"] });
   config.basePath = basePath;
   return config;
 }