chore: Partition example shredder to do one directory at a time

also exclude /dist/ folder and exclude _examples from jade shredder
closes #1661

gulpfile.js

@@ -615,7 +615,13 @@ gulp.task('_harp-compile', function() {
 });
 
 gulp.task('_shred-devguide-examples', ['_shred-clean-devguide', '_copy-example-boilerplate'], function() {
-  return docShredder.shred( _devguideShredOptions);
+  // Split big shredding task into partials 2016-06-14
+  var examplePaths = globby.sync(EXAMPLES_PATH+'/*/', {ignore: ['/node_modules', 'typings/', '_protractor/']});
+  var promise = Promise.resolve(true);
+  examplePaths.forEach(function (examplePath) {
+    promise = promise.then(() => docShredder.shredSingleExampleDir(_devguideShredOptions, examplePath));
+  });
+  return promise;
 });
 
 gulp.task('_shred-devguide-shared-jade', ['_shred-clean-devguide-shared-jade', '_copy-example-boilerplate'],  function() {
@@ -978,7 +984,7 @@ function devGuideExamplesWatch(shredOptions, postShredAction) {
   // removed this version because gulp.watch has the same glob issue that dgeni has.
   // var excludePattern = '!' + path.join(shredOptions.examplesDir, '**/node_modules/**/*.*');
   // gulp.watch([includePattern, excludePattern], {readDelay: 500}, function (event, done) {
-  var ignoreThese = [ '**/node_modules/**', '**/_fragments/**',
+  var ignoreThese = [ '**/node_modules/**', '**/_fragments/**', '**/dist/**', '**/typings/**',
                       '**/dart/.pub/**', '**/dart/build/**', '**/dart/packages/**'];
   var files = globby.sync( [includePattern], { ignore: ignoreThese });
   gulp.watch([files], {readDelay: 500}, function (event, done) {
@@ -994,7 +1000,7 @@ function devGuideSharedJadeWatch(shredOptions, postShredAction) {
   // removed this version because gulp.watch has the same glob issue that dgeni has.
   // var excludePattern = '!' + path.join(shredOptions.jadeDir, '**/node_modules/**/*.*');
   // gulp.watch([includePattern, excludePattern], {readDelay: 500}, function (event, done) {
-  var files = globby.sync( [includePattern], { ignore: [ '**/node_modules/**', '**/_fragments/**']});
+  var files = globby.sync( [includePattern], { ignore: [ '**/node_modules/**', '**/_examples/**', '**/_fragments/**']});
   gulp.watch([files], {readDelay: 500}, function (event, done) {
     gutil.log('Dev Guide jade file changed')
     gutil.log('Event type: ' + event.type); // added, changed, or deleted

tools/doc-shredder/doc-shredder.js

@@ -4,6 +4,7 @@ var del = require('del');
 var Dgeni = require('dgeni');
 var _ = require('lodash');
 var globby = require('globby');
+var ignoreDirs = ['**/node_modules/**', '**/dist/**', '**/typings/**'];
 
 var shred = function(shredOptions) {
   try {
@@ -22,6 +23,23 @@ var shred = function(shredOptions) {
   }
 }
 
+var shredSingleExampleDir = function(shredOptions, fileDir) {
+  shredOptions = resolveShredOptions(shredOptions);
+  var relativePath = path.relative(shredOptions.examplesDir, fileDir);
+  var examplesDir = path.join(shredOptions.examplesDir, relativePath);
+  var fragmentsDir = path.join(shredOptions.fragmentsDir, relativePath);
+  var options = {
+    includeSubdirs: true,
+    examplesDir: examplesDir,
+    fragmentsDir: fragmentsDir
+  }
+  var cleanPath = path.join(fragmentsDir, '*.*')
+  return del([ cleanPath, '!**/*.ovr.*']).then(function(paths) {
+    // console.log('Deleted files/folders:\n', paths.join('\n'));
+    return shred(options);
+  });
+}
+
 var shredSingleDir = function(shredOptions, filePath) {
   shredOptions = resolveShredOptions(shredOptions);
   var fileDir = path.dirname(filePath);
@@ -72,6 +90,7 @@ var buildShredMap = function(shredMapOptions) {
 
 module.exports = {
   shred: shred,
+  shredSingleExampleDir: shredSingleExampleDir,
   shredSingleDir: shredSingleDir,
   shredSingleJadeDir: shredSingleJadeDir,
   buildShredMap: buildShredMap
@@ -107,13 +126,14 @@ function createShredExamplePackage(shredOptions) {
 
       // HACK ( next two lines) because the glob function that dgeni uses internally isn't good at removing 'node_modules' early
       // this just uses globby to 'preglob' the include files ( and  exclude the node_modules).
-      var nmPattern = '**/node_modules/**';
+      var includeFiles = globby.sync( includeFiles, { ignore: ignoreDirs } );
-      var includeFiles = globby.sync( includeFiles, { ignore: [nmPattern] } );
+
+      console.log(`Shredding ${includeFiles.length} files inside ${shredOptions.examplesDir}`);
 
       readFilesProcessor.sourceFiles = [ {
         // Process all candidate files in `src` and its subfolders ...
         include: includeFiles ,
-        exclude: [ '**/node_modules/**', '**/typings/**', '**/packages/**', '**/build/**'],
+        exclude: [ '**/node_modules/**', '**/dist/**', '**/typings/**', '**/packages/**', '**/build/**'],
         // When calculating the relative path to these files use this as the base path.
         // So `src/foo/bar.js` will have relative path of `foo/bar.js`
         basePath: options.examplesDir
@@ -159,8 +179,7 @@ function createShredJadePackage(shredOptions) {
 
       // HACK ( next two lines) because the glob function that dgeni uses internally isn't good at removing 'node_modules' early
       // this just uses globby to 'preglob' the include files ( and  exclude the node_modules).
-      var nmPattern = '**/node_modules/**';
+      var includeFiles = globby.sync( includeFiles, { ignore: ignoreDirs } );
-      var includeFiles = globby.sync( includeFiles, { ignore: [nmPattern] } );
 
       readFilesProcessor.sourceFiles = [ {
         // Process all candidate files in `src` and its subfolders ...
@@ -212,8 +231,7 @@ var createShredMapPackage = function(mapOptions) {
 
       // HACK ( next two lines) because the glob function that dgeni uses internally isn't good at removing 'node_modules' early
       // this just uses globby to 'preglob' the include files ( and  exclude the node_modules).
-      var nmPattern = '**/node_modules/**';
+      var includeFiles = globby.sync( includeFiles, { ignore: ignoreDirs } );
-      var includeFiles = globby.sync( includeFiles, { ignore: [nmPattern] } );
 
 
       readFilesProcessor.sourceFiles = [ {