Work on zipping folders based on zipconfig.json files. Also involves refactoring of doc-shredder.

.gitignore

@@ -14,6 +14,7 @@ pubspec.lock
 **/ts/latest/api
 **/docs/_fragments
 public/docs/xref-*.*
+_zip-output
 
 
 

gulpfile.js

@@ -23,18 +23,22 @@ var prompt = require('prompt');
 
 
 var docShredder = require('./public/doc-shredder/doc-shredder');
+var exampleZipper = require('./public/_example-zipper/exampleZipper');
 
 var _devguideShredOptions =  {
   examplesDir: './public/docs/_examples',
-  fragmentsDir: './public/docs/_fragments'
+  fragmentsDir: './public/docs/_fragments',
+  zipDir: './public/resources/zips'
 };
 
 var _apiShredOptions =  {
   examplesDir: '../angular/modules/angular2/examples',
-  fragmentsDir: './public/docs/_fragments/_api'
+  fragmentsDir: './public/docs/_fragments/_api',
+  zipDir: './public/resources/zips/api'
 };
 
 
+
 var _excludePatterns = ['**/node_modules/**', '**/typings/**', '**/packages/**'];
 
 var _excludeMatchers = _excludePatterns.map(function(excludePattern){
@@ -85,7 +89,7 @@ gulp.task('build-and-serve', ['build-docs'], function (cb) {
   });
 });
 
-gulp.task('build-docs', ['_shred-devguide-examples', 'build-api-docs'], function() {
+gulp.task('build-docs', ['_shred-devguide-examples', 'build-api-docs', '_zip-examples'], function() {
   return buildShredMaps(true);
 });
 
@@ -122,6 +126,11 @@ gulp.task('_build-shred-maps', function() {
   return build-shred-maps(true);
 });
 
+gulp.task('_zip-examples', function() {
+  exampleZipper.zipExamples(_devguideShredOptions.examplesDir, _devguideShredOptions.zipDir);
+  exampleZipper.zipExamples(_apiShredOptions.examplesDir, _apiShredOptions.zipDir);
+});
+
 gulp.task('git-changed-examples', ['_shred-devguide-examples'], function(){
   var after, sha, messageSuffix;
   if (argv.after) {

package.json

@@ -24,6 +24,8 @@
     "url": ""
   },
   "devDependencies": {
+    "archiver": "^0.16.0",
+    "assert-plus": "^0.1.5",
     "browser-sync": "^2.9.3",
     "canonical-path": "0.0.2",
     "del": "^1.2.0",
@@ -50,6 +52,7 @@
     "lodash": "^3.10.1",
     "marked": "^0.3.5",
     "minimatch": "^2.0.10",
+    "mkdirp": "^0.5.1",
     "node-html-encoder": "0.0.2",
     "nodegit": "0.5.0",
     "path": "^0.11.14",

public/_example-zipper/Ideas.md

@@ -0,0 +1,15 @@
+Summary:
+        
+        1) if we discover a 'zipconfig.json' file or an 'xxx.zipconfig.json' file with the following structure 
+
+          {
+            "zipRegion": "zip",
+            "files": [ "foo.js", "**/**/.css", "!xxx.css"]
+          }
+          
+          where "zipRegion" is optional.
+          
+          Then we zip up all of the files specified, cleaning and removing extra doc tags.  If the specified 'zipRegion' 
+          is discovered in any file then we only zip that region. Otherwise we zip the entire file.
+          
+          

public/_example-zipper/exampleZipper.js

@@ -0,0 +1,98 @@
+// Canonical path provides a consistent path (i.e. always forward slashes) across different OSes
+var path = require('canonical-path');
+var Q = require('q');
+var _ = require('lodash');
+var jsonfile = require('jsonfile');
+var assert = require('assert-plus');
+// adm-zip does not work properly on Windows
+// var Zip = require('adm-zip');
+var archiver = require('archiver');
+
+var fs = require('fs');
+var mkdirp = require('mkdirp');
+
+var globule = require('globule');
+
+var regionExtractor = require('../doc-shredder/regionExtractor');
+
+
+// globs is an array of globs
+function zipExamples(sourceDirName, outputDirName) {
+  var gpath = path.join(sourceDirName, '**/*zipconfig.json');
+  var configFileNames = globule.find([gpath]);
+  configFileNames.forEach(function(configFileName) {
+    zipExample(configFileName, sourceDirName, outputDirName);
+  });
+}
+
+function zipExample(configFileName, sourceDirName, outputDirName) {
+  var json = jsonfile.readFileSync(configFileName);
+  var fileGlobs = json.files;
+  var zipRegionName = json.zipRegion;
+
+  // assert that fileGlobs is an array
+  assert.arrayOfString(fileGlobs, 'files property should be an array of strings');
+
+  // backup two from the relative configFileName
+  var relativeDirName = path.dirname(path.dirname(path.relative(sourceDirName, configFileName)));
+  var configDirName = path.dirname(configFileName);
+  // use the dir name of the folder containing the config file as the base file name of the zip file
+  var baseFileName = path.basename(configDirName);
+  // check if there is a prefix name before zipconfig.json
+  if (configFileName.indexOf('.zipconfig.json') >= 0) {
+    // if so use it as a suffix to the baseFileName
+    var extraName = path.basename(configFileName, '.zipconfig.json');
+    baseFileName = baseFileName + "." + extraName;
+  }
+
+  var outputFileName = path.join(outputDirName, relativeDirName, baseFileName + ".zip");
+  var fileNames = globule.find(fileGlobs, { srcBase: configDirName, prefixBase: configDirName });
+
+  var zip = createZipArchive(outputFileName);
+  fileNames.forEach(function(fileName) {
+    var relativePath = path.relative(configDirName, fileName);
+    var content = fs.readFileSync(fileName, 'utf8');
+    var extn = path.extname(fileName).substr(1);
+    // if we don't need to clean up the file then we can do the following.
+    // zip.append(fs.createReadStream(fileName), { name: relativePath });
+    var output;
+    // if no zipRegion or zipRegion is not in content then just clean the content
+    if (zipRegionName != null) {
+      output = regionExtractor.getRegionDoc(content, extn, zipRegionName);
+    }
+    if (!output) {
+      output = regionExtractor.removeDocTags(content, extn);
+    }
+
+    zip.append(output, { name: relativePath } )
+  });
+
+  zip.finalize();
+}
+
+function createZipArchive(zipFileName) {
+  var dirName = path.dirname(zipFileName);
+  // insure that the folder exists.
+  if (!fs.existsSync(dirName)) {
+    mkdirp.sync(dirName);
+  }
+  var output = fs.createWriteStream(zipFileName);
+  var archive = archiver('zip');
+
+  output.on('close', function () {
+    console.log('zip created: ' + zipFileName + ' (' + archive.pointer() + ' total bytes)');
+  });
+
+  archive.on('error', function (err) {
+    throw err;
+  });
+
+  archive.pipe(output);
+  return archive;
+}
+
+
+module.exports = {
+  zipExamples: zipExamples,
+  zipExample: zipExample
+};

public/_example-zipper/test/gulpfile.js

@@ -0,0 +1,26 @@
+var gulp = require('gulp');
+var path = require('canonical-path');
+var del = require('del');
+var taskListing = require('gulp-task-listing');
+
+var exampleZipper = require('../exampleZipper');
+
+var _outputFolder = '_zip-output';
+
+gulp.task('help', taskListing);
+
+gulp.task('zipExamples', ['clean'], function() {
+  return exampleZipper.zipExamples("../../docs/_examples", _outputFolder);
+});
+
+gulp.task('clean', function (cb) {
+  var cleanPath = path.join(_outputFolder, '**/*.*');
+  del([ cleanPath, '!**/*.ovr.*'], function (err, paths) {
+    // console.log('Deleted files/folders:\n', paths.join('\n'));
+    cb();
+  });
+});
+
+
+gulp.task('default', taskListing);
+

public/doc-shredder/doc-shredder.js

@@ -63,7 +63,6 @@ function createShredPackage(shredOptions) {
 
   initializePackage(pkg)
     .factory(require('./fileShredder'))
-    .factory(require('./regionExtractor'))
     .processor(require('./mdWrapperProcessor'))
 
     .config(function(readFilesProcessor, fileShredder ) {

public/doc-shredder/fileShredder.js

@@ -1,57 +1,21 @@
+var regionExtractor = require('./regionExtractor');
+var buildRegionDocs = regionExtractor.buildRegionDocs;
+
 /**
  * @dgService htmlFileShredder
  * @description
  */
-module.exports = function fileShredder(log, regionExtractor) {
+module.exports = function fileShredder(log ) {
   return {
     name: 'fileShredder',
 
     getDocs: function (fileInfo) {
-      var commentInfo;
-      switch (fileInfo.extension) {
-        case 'ts':
-        case 'js':
-        case 'dart':
-          commentInfo = {
-            prefix: '//',
-            blockPattern: '/* {tag} */'
-          };
-          //commentMarkers = ['//'];
-          break;
-        case 'html':
-          commentInfo = {
-            prefix: '<!--',
-            blockPattern: '<!-- {tag} -->'
-          };
-          // commentMarkers = ['<!--'];
-          break;
-        case 'css':
-          commentInfo = {
-            prefix: '/*',
-            blockPattern: '/* {tag} */'
-          };
-          // commentMarkers = ['/*'];
-          break;
-        case 'json':
-          break;
-        case 'yaml':
-          commentInfo = {
-            prefix: '#',
-            blockPattern: '# {tag} '
-          };
-          // commentMarkers = ['#'];
-          break;
-        default:
-          return {};
-      }
-      var docs;
       // log.info("fileShredder processing: " + fileInfo.relativePath);
-      if (commentInfo) {
+      var docs = buildRegionDocs(fileInfo.content, fileInfo.extension);
-        docs = regionExtractor(fileInfo.content, commentInfo);
+      var wasShredded = docs.some(function(doc) {
-      } else {
+        return doc.regionName != null;
-        docs = [ { content: fileInfo.content } ];
+      });
-      }
+      if (wasShredded) {
-      if (docs.length) {
         log.info("shredded file: " + fileInfo.relativePath);
       }
       return docs;

public/doc-shredder/mdWrapperProcessor.js

@@ -1,6 +1,3 @@
-// Not currently used - but wanted to leave it as an example
-// var _ = require('lodash');
-
 /**
  * dgProcessor shredderProcessor
  * @description

public/doc-shredder/regionExtractor.js

@@ -1,30 +1,43 @@
+/**
+ * NOT a dgeni service because we need to be able to use it externally
+ * as well as from dgeni.
+ * @description
+ *
+ */
 var _ = require('lodash');
 
-module.exports = function regionExtractor() {
+var nullLine = '###';
+var nullLinePattern = new RegExp(nullLine + '\n', 'g');
 
-  var nullLine = '###';
+module.exports = {
-  var nullLinePattern = new RegExp(nullLine + '\n', 'g');
+  buildRegionDocs: buildRegionDocs,
+  getRegionDoc: getRegionDoc,
+  removeDocTags: removeDocTags
+};
 
-  // split out each fragment in {content} into a separate doc
+// split out each fragment in {content} into a separate doc
-  // a fragment is a section of text surrounded by
+// a fragment is a section of text surrounded by
-  //   1) In front: a comment marker followed by '#docregion' followed by an optional region name. For example:
+//   1) In front: a comment marker followed by '#docregion' followed by an optional region name. For example:
-  //        <-- #docregion foo --> for html
+//        <-- #docregion foo --> for html
-  //     or // #docregion foo      for js/ts
+//     or // #docregion foo      for js/ts
-  //   2) In back: a comment marker followed by '#enddocregion'
+//   2) In back: a comment marker followed by '#enddocregion'
-  // Regions can be nested and any regions not 'closed' are automatically closed at the end of the doc.
+// Regions can be nested and any regions not 'closed' are automatically closed at the end of the doc.
 
-  // empty enddocregion always closes last region started.
+// empty enddocregion always closes last region started.
-  // enddocregions with names that do no match start region tags get ignored.
+// enddocregions with names that do no match start region tags get ignored.
-
+function buildRegionDocs(content, extn) {
-  return function(content, commentInfo) {
+  var commentInfo = getCommentInfo(extn);
+  if (!commentInfo) {
+    return  [ { content: content } ];
+  }
 
   var lines = result = content.split(/\r?\n/);
 
   var docStack = []; // items will be both popped and removed from the middle
   var docMap = {};
+  var docPlaster = '. . .';
   var doc;
   var regionNames;
-    var docPlaster = '. . .';
   lines.forEach(function(line, ix) {
     if (isCommentLine(line, commentInfo.prefix)) {
       if (hasRegionTag(line)) {
@@ -78,7 +91,39 @@ module.exports = function regionExtractor() {
   });
 
   var docs = _.values(docMap);
-    var plasterComment = docPlaster && commentInfo.blockPattern.replace('{tag}', docPlaster);
+  var plasterComment = docPlaster && commentInfo.plasterPattern.replace('{tag}', docPlaster);
+  docs = reattachDocs(docs, lines, plasterComment);
+  return docs;
+}
+
+function getRegionDoc(content, extn, regionName) {
+  var docs = buildRegionDocs(content, extn);
+  var doc = _.find(docs, function (doc) {
+    return doc.regionName === regionName;
+  });
+  return doc && doc.content;
+}
+
+function removeDocTags(content, extn) {
+  var commentInfo = getCommentInfo(extn);
+  if (commentInfo == null) {
+    return content;
+  }
+  var lines = result = content.split(/\r?\n/);
+
+  lines.forEach(function(line, ix) {
+    if (isCommentLine(line, commentInfo.prefix)) {
+      if (hasDocTag(line)) {
+        lines[ix] = nullLine;
+      }
+    }
+  });
+  var result = joinLines(lines);
+  return result;
+}
+
+function reattachDocs(docs, lines, plasterComment) {
+
   docs.forEach(function(doc) {
     var content;
     var fragLines = [];
@@ -97,48 +142,58 @@ module.exports = function regionExtractor() {
       fragLines = fragLines.concat(subLines);
     });
     fragLines = trimLeftIndent(fragLines);
-      content = fragLines.join('\n');
+    doc.content = joinLines(fragLines);
-      // eliminate all #docregion lines
-      content = content.replace(nullLinePattern, '');
-      if (content.substr(-3) === nullLine) {
-        content = content.substr(0, content.length-3);
-      }
-      doc.content = content;
   });
   return docs;
-  }
+}
 
-};
 
-function trimLeftIndent(lines) {
-  var minIx  = 100;
-  var ok = lines.every(function(line) {
-    // var ix = line.search(/\S/);
-    var ix = line.search(/[^ ]/);
-    if (ix === 0) return false;
-    if (ix === -1) return true;
-    if (ix > 0) {
-      minIx = Math.min(minIx, ix);
-    }
-    return true;
-  });
-  if ( (!ok) || minIx === 100) return lines;
 
-  var result = lines.map(function(line) {
+function getCommentInfo(extension) {
-    if (line.length > minIx) {
+  var commentInfo;
-      return line.substr(minIx);
+  switch (extension) {
-    } else {
+    case 'ts':
-      // this can happen if line is all blanks and shorter than mixIx
+    case 'js':
-      return line;
+    case 'dart':
+      commentInfo = {
+        prefix: '//',
+        plasterPattern: '/* {tag} */'
+      };
+      break;
+    case 'html':
+      commentInfo = {
+        prefix: '<!--',
+        plasterPattern: '<!-- {tag} -->'
+      };
+      break;
+    case 'css':
+      commentInfo = {
+        prefix: '/*',
+        plasterPattern: '/* {tag} */'
+      };
+      break;
+    case 'json':
+      return null;
+    case 'yaml':
+      commentInfo = {
+        prefix: '#',
+        plasterPattern: '# {tag} '
+      };
+      break;
+    default:
+      return null;
   }
-  });
+  return commentInfo;
-  return result;
 }
 
 function isCommentLine(line, commentPrefix) {
   return line.trim().indexOf(commentPrefix) == 0;
 }
 
+function hasDocTag(line) {
+  return hasRegionTag(line) || hasEndRegionTag(line) || hasDocPlasterTag(line);
+}
+
 function hasRegionTag(line) {
   return line.indexOf("#docregion") >= 0;
 }
@@ -181,3 +236,38 @@ function extractRegionNames(line, rx) {
     return [''];
   }
 }
+
+function trimLeftIndent(lines) {
+  var minIx  = 100;
+  var ok = lines.every(function(line) {
+    // var ix = line.search(/\S/);
+    var ix = line.search(/[^ ]/);
+    if (ix === 0) return false;
+    if (ix === -1) return true;
+    if (ix > 0) {
+      minIx = Math.min(minIx, ix);
+    }
+    return true;
+  });
+  if ( (!ok) || minIx === 100) return lines;
+
+  var result = lines.map(function(line) {
+    if (line.length > minIx) {
+      return line.substr(minIx);
+    } else {
+      // this can happen if line is all blanks and shorter than mixIx
+      return line;
+    }
+  });
+  return result;
+}
+
+function joinLines(lines) {
+  var content = lines.join('\n');
+  // eliminate all #docregion lines
+  content = content.replace(nullLinePattern, '');
+  if (content.substr(-3) === nullLine) {
+    content = content.substr(0, content.length - 3);
+  }
+  return content;
+}

public/doc-shredder/regionExtractor.old.js

@@ -1,106 +0,0 @@
-module.exports = function regionExtractor() {
-
-  var nullLine = '###';
-  var nullLinePattern = new RegExp(nullLine + '\n', 'g');
-
-  // split out each fragment in {content} into a separate doc
-  // a fragment is a section of text surrounded by
-  //   1) In front: a comment marker followed by '#docregion' followed by an optional region name. For example:
-  //        <-- #docregion foo --> for html
-  //     or // #docregion foo      for js/ts
-  //   2) In back: a comment marker followed by '#enddocregion'
-  // Regions can be nested and any regions not 'closed' are automatically closed at the end of the doc.
-  return function(content, commentPrefixes) {
-
-    var lines = result = content.split(/\r?\n/);
-    var docs = [];
-    var docStack = [];
-    var doc = null;
-
-    lines.forEach(function(line, ix) {
-      if (isCommentLine(line, commentPrefixes)) {
-        if (hasRegionTag(line)) {
-          if (doc) docStack.push(doc);
-          doc = {startIx: ix, regionName: getRegionName(line)};
-          lines[ix] = nullLine;
-          docs.push(doc);
-        } else if (hasEndRegionTag(line)) {
-          if (doc) {
-            lines[ix] = nullLine;
-            doc.endIx = ix;
-            doc = docStack.pop();
-          }
-        }
-      }
-    });
-
-    docs.forEach(function(doc) {
-      var fragLines, content;
-      if (doc.endIx) {
-        fragLines = lines.slice(doc.startIx + 1, doc.endIx);
-      } else {
-        fragLines = lines.slice(doc.startIx + 1);
-      }
-      fragLines = trimLeftIndent(fragLines);
-      content = fragLines.join('\n');
-      // eliminate all #docregion lines
-      content = content.replace(nullLinePattern, '');
-      if (content.substr(-3) === nullLine) {
-        content = content.substr(0, content.length-3);
-      }
-      doc.content = content;
-    });
-    return docs;
-  }
-
-};
-
-function trimLeftIndent(lines) {
-  var minIx  = 100;
-  var ok = lines.every(function(line) {
-    // var ix = line.search(/\S/);
-    var ix = line.search(/[^ ]/);
-    if (ix === 0) return false;
-    if (ix === -1) return true;
-    if (ix > 0) {
-      minIx = Math.min(minIx, ix);
-    }
-    return true;
-  });
-  if ( (!ok) || minIx === 100) return lines;
-
-  var result = lines.map(function(line) {
-    if (line.length > minIx) {
-      return line.substr(minIx);
-    } else {
-      // this can happen if line is all blanks and shorter than mixIx
-      return line;
-    }
-  });
-  return result;
-}
-
-function isCommentLine(line, commentPrefixes) {
-  return commentPrefixes.some(function(prefix) {
-    return line.trim().indexOf(prefix) == 0;
-  });
-}
-
-function hasRegionTag(line) {
-  return line.indexOf("#docregion") >= 0;
-}
-
-function hasEndRegionTag(line) {
-  return line.indexOf("#enddocregion") >= 0;
-}
-
-function getRegionName(line) {
-  try {
-    var name = line.match(/#docregion\s*(\S*).*/)[1];
-    // Hack for html regions that look like <!-- #docregion --> or */
-    name = name.replace("-->","").replace('\*\/',"");
-    return name;
-  } catch (e) {
-    return '';
-  }
-}

public/docs/_examples/styleguide/jsonly.zipconfig.json

@@ -0,0 +1,3 @@
+{
+  "files": ["**/*.js"]
+}

public/docs/_examples/styleguide/zipconfig.json

@@ -0,0 +1,4 @@
+{
+  "zipRegion": "class",
+  "files": ["**/*.*", "!**/*zipconfig.json"]
+}