Work on zipping folders based on zipconfig.json files. Also involves refactoring of doc-shredder.

This commit is contained in:
Jay Traband 2015-10-23 12:47:20 -07:00 committed by Naomi Black
parent 840439770b
commit 9500a123f5
15 changed files with 387 additions and 284 deletions

1
.gitignore vendored
View File

@ -14,6 +14,7 @@ pubspec.lock
**/ts/latest/api **/ts/latest/api
**/docs/_fragments **/docs/_fragments
public/docs/xref-*.* public/docs/xref-*.*
_zip-output

View File

@ -23,18 +23,22 @@ var prompt = require('prompt');
var docShredder = require('./public/doc-shredder/doc-shredder'); var docShredder = require('./public/doc-shredder/doc-shredder');
var exampleZipper = require('./public/_example-zipper/exampleZipper');
var _devguideShredOptions = { var _devguideShredOptions = {
examplesDir: './public/docs/_examples', examplesDir: './public/docs/_examples',
fragmentsDir: './public/docs/_fragments' fragmentsDir: './public/docs/_fragments',
zipDir: './public/resources/zips'
}; };
var _apiShredOptions = { var _apiShredOptions = {
examplesDir: '../angular/modules/angular2/examples', examplesDir: '../angular/modules/angular2/examples',
fragmentsDir: './public/docs/_fragments/_api' fragmentsDir: './public/docs/_fragments/_api',
zipDir: './public/resources/zips/api'
}; };
var _excludePatterns = ['**/node_modules/**', '**/typings/**', '**/packages/**']; var _excludePatterns = ['**/node_modules/**', '**/typings/**', '**/packages/**'];
var _excludeMatchers = _excludePatterns.map(function(excludePattern){ var _excludeMatchers = _excludePatterns.map(function(excludePattern){
@ -85,7 +89,7 @@ gulp.task('build-and-serve', ['build-docs'], function (cb) {
}); });
}); });
gulp.task('build-docs', ['_shred-devguide-examples', 'build-api-docs'], function() { gulp.task('build-docs', ['_shred-devguide-examples', 'build-api-docs', '_zip-examples'], function() {
return buildShredMaps(true); return buildShredMaps(true);
}); });
@ -122,6 +126,11 @@ gulp.task('_build-shred-maps', function() {
return build-shred-maps(true); return build-shred-maps(true);
}); });
gulp.task('_zip-examples', function() {
exampleZipper.zipExamples(_devguideShredOptions.examplesDir, _devguideShredOptions.zipDir);
exampleZipper.zipExamples(_apiShredOptions.examplesDir, _apiShredOptions.zipDir);
});
gulp.task('git-changed-examples', ['_shred-devguide-examples'], function(){ gulp.task('git-changed-examples', ['_shred-devguide-examples'], function(){
var after, sha, messageSuffix; var after, sha, messageSuffix;
if (argv.after) { if (argv.after) {

View File

@ -24,6 +24,8 @@
"url": "" "url": ""
}, },
"devDependencies": { "devDependencies": {
"archiver": "^0.16.0",
"assert-plus": "^0.1.5",
"browser-sync": "^2.9.3", "browser-sync": "^2.9.3",
"canonical-path": "0.0.2", "canonical-path": "0.0.2",
"del": "^1.2.0", "del": "^1.2.0",
@ -50,6 +52,7 @@
"lodash": "^3.10.1", "lodash": "^3.10.1",
"marked": "^0.3.5", "marked": "^0.3.5",
"minimatch": "^2.0.10", "minimatch": "^2.0.10",
"mkdirp": "^0.5.1",
"node-html-encoder": "0.0.2", "node-html-encoder": "0.0.2",
"nodegit": "0.5.0", "nodegit": "0.5.0",
"path": "^0.11.14", "path": "^0.11.14",

View File

@ -0,0 +1,15 @@
Summary:
1) if we discover a 'zipconfig.json' file or an 'xxx.zipconfig.json' file with the following structure
{
"zipRegion": "zip",
"files": [ "foo.js", "**/**/.css", "!xxx.css"]
}
where "zipRegion" is optional.
Then we zip up all of the files specified, cleaning and removing extra doc tags. If the specified 'zipRegion'
is discovered in any file then we only zip that region. Otherwise we zip the entire file.

View File

@ -0,0 +1,98 @@
// Canonical path provides a consistent path (i.e. always forward slashes) across different OSes
var path = require('canonical-path');
var Q = require('q');
var _ = require('lodash');
var jsonfile = require('jsonfile');
var assert = require('assert-plus');
// adm-zip does not work properly on Windows
// var Zip = require('adm-zip');
var archiver = require('archiver');
var fs = require('fs');
var mkdirp = require('mkdirp');
var globule = require('globule');
var regionExtractor = require('../doc-shredder/regionExtractor');
// globs is an array of globs
function zipExamples(sourceDirName, outputDirName) {
var gpath = path.join(sourceDirName, '**/*zipconfig.json');
var configFileNames = globule.find([gpath]);
configFileNames.forEach(function(configFileName) {
zipExample(configFileName, sourceDirName, outputDirName);
});
}
function zipExample(configFileName, sourceDirName, outputDirName) {
var json = jsonfile.readFileSync(configFileName);
var fileGlobs = json.files;
var zipRegionName = json.zipRegion;
// assert that fileGlobs is an array
assert.arrayOfString(fileGlobs, 'files property should be an array of strings');
// backup two from the relative configFileName
var relativeDirName = path.dirname(path.dirname(path.relative(sourceDirName, configFileName)));
var configDirName = path.dirname(configFileName);
// use the dir name of the folder containing the config file as the base file name of the zip file
var baseFileName = path.basename(configDirName);
// check if there is a prefix name before zipconfig.json
if (configFileName.indexOf('.zipconfig.json') >= 0) {
// if so use it as a suffix to the baseFileName
var extraName = path.basename(configFileName, '.zipconfig.json');
baseFileName = baseFileName + "." + extraName;
}
var outputFileName = path.join(outputDirName, relativeDirName, baseFileName + ".zip");
var fileNames = globule.find(fileGlobs, { srcBase: configDirName, prefixBase: configDirName });
var zip = createZipArchive(outputFileName);
fileNames.forEach(function(fileName) {
var relativePath = path.relative(configDirName, fileName);
var content = fs.readFileSync(fileName, 'utf8');
var extn = path.extname(fileName).substr(1);
// if we don't need to clean up the file then we can do the following.
// zip.append(fs.createReadStream(fileName), { name: relativePath });
var output;
// if no zipRegion or zipRegion is not in content then just clean the content
if (zipRegionName != null) {
output = regionExtractor.getRegionDoc(content, extn, zipRegionName);
}
if (!output) {
output = regionExtractor.removeDocTags(content, extn);
}
zip.append(output, { name: relativePath } )
});
zip.finalize();
}
function createZipArchive(zipFileName) {
var dirName = path.dirname(zipFileName);
// insure that the folder exists.
if (!fs.existsSync(dirName)) {
mkdirp.sync(dirName);
}
var output = fs.createWriteStream(zipFileName);
var archive = archiver('zip');
output.on('close', function () {
console.log('zip created: ' + zipFileName + ' (' + archive.pointer() + ' total bytes)');
});
archive.on('error', function (err) {
throw err;
});
archive.pipe(output);
return archive;
}
module.exports = {
zipExamples: zipExamples,
zipExample: zipExample
};

View File

@ -0,0 +1,26 @@
var gulp = require('gulp');
var path = require('canonical-path');
var del = require('del');
var taskListing = require('gulp-task-listing');
var exampleZipper = require('../exampleZipper');
var _outputFolder = '_zip-output';
gulp.task('help', taskListing);
gulp.task('zipExamples', ['clean'], function() {
return exampleZipper.zipExamples("../../docs/_examples", _outputFolder);
});
gulp.task('clean', function (cb) {
var cleanPath = path.join(_outputFolder, '**/*.*');
del([ cleanPath, '!**/*.ovr.*'], function (err, paths) {
// console.log('Deleted files/folders:\n', paths.join('\n'));
cb();
});
});
gulp.task('default', taskListing);

View File

@ -63,7 +63,6 @@ function createShredPackage(shredOptions) {
initializePackage(pkg) initializePackage(pkg)
.factory(require('./fileShredder')) .factory(require('./fileShredder'))
.factory(require('./regionExtractor'))
.processor(require('./mdWrapperProcessor')) .processor(require('./mdWrapperProcessor'))
.config(function(readFilesProcessor, fileShredder ) { .config(function(readFilesProcessor, fileShredder ) {

View File

@ -1,57 +1,21 @@
var regionExtractor = require('./regionExtractor');
var buildRegionDocs = regionExtractor.buildRegionDocs;
/** /**
* @dgService htmlFileShredder * @dgService htmlFileShredder
* @description * @description
*/ */
module.exports = function fileShredder(log, regionExtractor) { module.exports = function fileShredder(log ) {
return { return {
name: 'fileShredder', name: 'fileShredder',
getDocs: function (fileInfo) { getDocs: function (fileInfo) {
var commentInfo;
switch (fileInfo.extension) {
case 'ts':
case 'js':
case 'dart':
commentInfo = {
prefix: '//',
blockPattern: '/* {tag} */'
};
//commentMarkers = ['//'];
break;
case 'html':
commentInfo = {
prefix: '<!--',
blockPattern: '<!-- {tag} -->'
};
// commentMarkers = ['<!--'];
break;
case 'css':
commentInfo = {
prefix: '/*',
blockPattern: '/* {tag} */'
};
// commentMarkers = ['/*'];
break;
case 'json':
break;
case 'yaml':
commentInfo = {
prefix: '#',
blockPattern: '# {tag} '
};
// commentMarkers = ['#'];
break;
default:
return {};
}
var docs;
// log.info("fileShredder processing: " + fileInfo.relativePath); // log.info("fileShredder processing: " + fileInfo.relativePath);
if (commentInfo) { var docs = buildRegionDocs(fileInfo.content, fileInfo.extension);
docs = regionExtractor(fileInfo.content, commentInfo); var wasShredded = docs.some(function(doc) {
} else { return doc.regionName != null;
docs = [ { content: fileInfo.content } ]; });
} if (wasShredded) {
if (docs.length) {
log.info("shredded file: " + fileInfo.relativePath); log.info("shredded file: " + fileInfo.relativePath);
} }
return docs; return docs;

View File

@ -1,6 +1,3 @@
// Not currently used - but wanted to leave it as an example
// var _ = require('lodash');
/** /**
* dgProcessor shredderProcessor * dgProcessor shredderProcessor
* @description * @description

View File

@ -1,144 +1,199 @@
/**
* NOT a dgeni service because we need to be able to use it externally
* as well as from dgeni.
* @description
*
*/
var _ = require('lodash'); var _ = require('lodash');
module.exports = function regionExtractor() { var nullLine = '###';
var nullLinePattern = new RegExp(nullLine + '\n', 'g');
var nullLine = '###';
var nullLinePattern = new RegExp(nullLine + '\n', 'g');
// split out each fragment in {content} into a separate doc
// a fragment is a section of text surrounded by
// 1) In front: a comment marker followed by '#docregion' followed by an optional region name. For example:
// <-- #docregion foo --> for html
// or // #docregion foo for js/ts
// 2) In back: a comment marker followed by '#enddocregion'
// Regions can be nested and any regions not 'closed' are automatically closed at the end of the doc.
// empty enddocregion always closes last region started.
// enddocregions with names that do no match start region tags get ignored.
return function(content, commentInfo) {
var lines = result = content.split(/\r?\n/);
var docStack = []; // items will be both popped and removed from the middle
var docMap = {};
var doc;
var regionNames;
var docPlaster = '. . .';
lines.forEach(function(line, ix) {
if (isCommentLine(line, commentInfo.prefix)) {
if (hasRegionTag(line)) {
lines[ix] = nullLine;
regionNames = getRegionNames(line);
regionNames.forEach(function(rn) {
doc = docMap[rn];
if (!doc) {
// regionName may be ''
doc = {regionName: rn, ranges: [ { startIx: ix} ] };
docMap[rn] = doc;
} else {
// only add a new range if prev range is closed
var lastRange = doc.ranges[doc.ranges.length-1];
if (lastRange.endIx) {
doc.ranges.push({startIx: ix});
}
}
docStack.push(doc);
});
} else if (hasEndRegionTag(line)) {
lines[ix] = nullLine;
regionNames = getEndRegionNames(line);
regionNames.forEach(function(rn) {
// handle endregions with no name specially.
// They operate on the last region created.
if (rn.length == 0) {
if (docStack.length) {
// update last item on the stack
doc = docStack.pop();
doc.ranges[doc.ranges.length - 1].endIx = ix;
}
} else {
doc = docMap[rn];
// ignore endregion if name is specified but not found.
if (doc) {
doc.ranges[doc.ranges.length - 1].endIx = ix;
// remove doc from stack
_.remove(docStack, function (item) {
return item.regionName === rn;
});
}
}
});
} else if (hasDocPlasterTag(line)) {
line[ix] = nullLine;
docPlaster = getDocPlaster(line);
}
}
});
var docs = _.values(docMap);
var plasterComment = docPlaster && commentInfo.blockPattern.replace('{tag}', docPlaster);
docs.forEach(function(doc) {
var content;
var fragLines = [];
doc.ranges.forEach(function (range) {
var subLines;
if (range.endIx) {
subLines = lines.slice(range.startIx + 1, range.endIx);
} else {
subLines = lines.slice(range.startIx + 1);
}
if (plasterComment && fragLines.length) {
// pad is the padding on the previous line
var pad = fragLines[fragLines.length - 1].match(/(\s*)/)[0];
fragLines.push(pad + plasterComment);
}
fragLines = fragLines.concat(subLines);
});
fragLines = trimLeftIndent(fragLines);
content = fragLines.join('\n');
// eliminate all #docregion lines
content = content.replace(nullLinePattern, '');
if (content.substr(-3) === nullLine) {
content = content.substr(0, content.length-3);
}
doc.content = content;
});
return docs;
}
module.exports = {
buildRegionDocs: buildRegionDocs,
getRegionDoc: getRegionDoc,
removeDocTags: removeDocTags
}; };
function trimLeftIndent(lines) { // split out each fragment in {content} into a separate doc
var minIx = 100; // a fragment is a section of text surrounded by
var ok = lines.every(function(line) { // 1) In front: a comment marker followed by '#docregion' followed by an optional region name. For example:
// var ix = line.search(/\S/); // <-- #docregion foo --> for html
var ix = line.search(/[^ ]/); // or // #docregion foo for js/ts
if (ix === 0) return false; // 2) In back: a comment marker followed by '#enddocregion'
if (ix === -1) return true; // Regions can be nested and any regions not 'closed' are automatically closed at the end of the doc.
if (ix > 0) {
minIx = Math.min(minIx, ix);
}
return true;
});
if ( (!ok) || minIx === 100) return lines;
var result = lines.map(function(line) { // empty enddocregion always closes last region started.
if (line.length > minIx) { // enddocregions with names that do no match start region tags get ignored.
return line.substr(minIx); function buildRegionDocs(content, extn) {
} else { var commentInfo = getCommentInfo(extn);
// this can happen if line is all blanks and shorter than mixIx if (!commentInfo) {
return line; return [ { content: content } ];
}
var lines = result = content.split(/\r?\n/);
var docStack = []; // items will be both popped and removed from the middle
var docMap = {};
var docPlaster = '. . .';
var doc;
var regionNames;
lines.forEach(function(line, ix) {
if (isCommentLine(line, commentInfo.prefix)) {
if (hasRegionTag(line)) {
lines[ix] = nullLine;
regionNames = getRegionNames(line);
regionNames.forEach(function(rn) {
doc = docMap[rn];
if (!doc) {
// regionName may be ''
doc = {regionName: rn, ranges: [ { startIx: ix} ] };
docMap[rn] = doc;
} else {
// only add a new range if prev range is closed
var lastRange = doc.ranges[doc.ranges.length-1];
if (lastRange.endIx) {
doc.ranges.push({startIx: ix});
}
}
docStack.push(doc);
});
} else if (hasEndRegionTag(line)) {
lines[ix] = nullLine;
regionNames = getEndRegionNames(line);
regionNames.forEach(function(rn) {
// handle endregions with no name specially.
// They operate on the last region created.
if (rn.length == 0) {
if (docStack.length) {
// update last item on the stack
doc = docStack.pop();
doc.ranges[doc.ranges.length - 1].endIx = ix;
}
} else {
doc = docMap[rn];
// ignore endregion if name is specified but not found.
if (doc) {
doc.ranges[doc.ranges.length - 1].endIx = ix;
// remove doc from stack
_.remove(docStack, function (item) {
return item.regionName === rn;
});
}
}
});
} else if (hasDocPlasterTag(line)) {
line[ix] = nullLine;
docPlaster = getDocPlaster(line);
}
} }
}); });
var docs = _.values(docMap);
var plasterComment = docPlaster && commentInfo.plasterPattern.replace('{tag}', docPlaster);
docs = reattachDocs(docs, lines, plasterComment);
return docs;
}
function getRegionDoc(content, extn, regionName) {
var docs = buildRegionDocs(content, extn);
var doc = _.find(docs, function (doc) {
return doc.regionName === regionName;
});
return doc && doc.content;
}
function removeDocTags(content, extn) {
var commentInfo = getCommentInfo(extn);
if (commentInfo == null) {
return content;
}
var lines = result = content.split(/\r?\n/);
lines.forEach(function(line, ix) {
if (isCommentLine(line, commentInfo.prefix)) {
if (hasDocTag(line)) {
lines[ix] = nullLine;
}
}
});
var result = joinLines(lines);
return result; return result;
} }
function reattachDocs(docs, lines, plasterComment) {
docs.forEach(function(doc) {
var content;
var fragLines = [];
doc.ranges.forEach(function (range) {
var subLines;
if (range.endIx) {
subLines = lines.slice(range.startIx + 1, range.endIx);
} else {
subLines = lines.slice(range.startIx + 1);
}
if (plasterComment && fragLines.length) {
// pad is the padding on the previous line
var pad = fragLines[fragLines.length - 1].match(/(\s*)/)[0];
fragLines.push(pad + plasterComment);
}
fragLines = fragLines.concat(subLines);
});
fragLines = trimLeftIndent(fragLines);
doc.content = joinLines(fragLines);
});
return docs;
}
function getCommentInfo(extension) {
var commentInfo;
switch (extension) {
case 'ts':
case 'js':
case 'dart':
commentInfo = {
prefix: '//',
plasterPattern: '/* {tag} */'
};
break;
case 'html':
commentInfo = {
prefix: '<!--',
plasterPattern: '<!-- {tag} -->'
};
break;
case 'css':
commentInfo = {
prefix: '/*',
plasterPattern: '/* {tag} */'
};
break;
case 'json':
return null;
case 'yaml':
commentInfo = {
prefix: '#',
plasterPattern: '# {tag} '
};
break;
default:
return null;
}
return commentInfo;
}
function isCommentLine(line, commentPrefix) { function isCommentLine(line, commentPrefix) {
return line.trim().indexOf(commentPrefix) == 0; return line.trim().indexOf(commentPrefix) == 0;
} }
function hasDocTag(line) {
return hasRegionTag(line) || hasEndRegionTag(line) || hasDocPlasterTag(line);
}
function hasRegionTag(line) { function hasRegionTag(line) {
return line.indexOf("#docregion") >= 0; return line.indexOf("#docregion") >= 0;
} }
@ -181,3 +236,38 @@ function extractRegionNames(line, rx) {
return ['']; return [''];
} }
} }
function trimLeftIndent(lines) {
var minIx = 100;
var ok = lines.every(function(line) {
// var ix = line.search(/\S/);
var ix = line.search(/[^ ]/);
if (ix === 0) return false;
if (ix === -1) return true;
if (ix > 0) {
minIx = Math.min(minIx, ix);
}
return true;
});
if ( (!ok) || minIx === 100) return lines;
var result = lines.map(function(line) {
if (line.length > minIx) {
return line.substr(minIx);
} else {
// this can happen if line is all blanks and shorter than mixIx
return line;
}
});
return result;
}
function joinLines(lines) {
var content = lines.join('\n');
// eliminate all #docregion lines
content = content.replace(nullLinePattern, '');
if (content.substr(-3) === nullLine) {
content = content.substr(0, content.length - 3);
}
return content;
}

View File

@ -1,106 +0,0 @@
module.exports = function regionExtractor() {
var nullLine = '###';
var nullLinePattern = new RegExp(nullLine + '\n', 'g');
// split out each fragment in {content} into a separate doc
// a fragment is a section of text surrounded by
// 1) In front: a comment marker followed by '#docregion' followed by an optional region name. For example:
// <-- #docregion foo --> for html
// or // #docregion foo for js/ts
// 2) In back: a comment marker followed by '#enddocregion'
// Regions can be nested and any regions not 'closed' are automatically closed at the end of the doc.
return function(content, commentPrefixes) {
var lines = result = content.split(/\r?\n/);
var docs = [];
var docStack = [];
var doc = null;
lines.forEach(function(line, ix) {
if (isCommentLine(line, commentPrefixes)) {
if (hasRegionTag(line)) {
if (doc) docStack.push(doc);
doc = {startIx: ix, regionName: getRegionName(line)};
lines[ix] = nullLine;
docs.push(doc);
} else if (hasEndRegionTag(line)) {
if (doc) {
lines[ix] = nullLine;
doc.endIx = ix;
doc = docStack.pop();
}
}
}
});
docs.forEach(function(doc) {
var fragLines, content;
if (doc.endIx) {
fragLines = lines.slice(doc.startIx + 1, doc.endIx);
} else {
fragLines = lines.slice(doc.startIx + 1);
}
fragLines = trimLeftIndent(fragLines);
content = fragLines.join('\n');
// eliminate all #docregion lines
content = content.replace(nullLinePattern, '');
if (content.substr(-3) === nullLine) {
content = content.substr(0, content.length-3);
}
doc.content = content;
});
return docs;
}
};
function trimLeftIndent(lines) {
var minIx = 100;
var ok = lines.every(function(line) {
// var ix = line.search(/\S/);
var ix = line.search(/[^ ]/);
if (ix === 0) return false;
if (ix === -1) return true;
if (ix > 0) {
minIx = Math.min(minIx, ix);
}
return true;
});
if ( (!ok) || minIx === 100) return lines;
var result = lines.map(function(line) {
if (line.length > minIx) {
return line.substr(minIx);
} else {
// this can happen if line is all blanks and shorter than mixIx
return line;
}
});
return result;
}
function isCommentLine(line, commentPrefixes) {
return commentPrefixes.some(function(prefix) {
return line.trim().indexOf(prefix) == 0;
});
}
function hasRegionTag(line) {
return line.indexOf("#docregion") >= 0;
}
function hasEndRegionTag(line) {
return line.indexOf("#enddocregion") >= 0;
}
function getRegionName(line) {
try {
var name = line.match(/#docregion\s*(\S*).*/)[1];
// Hack for html regions that look like <!-- #docregion --> or */
name = name.replace("-->","").replace('\*\/',"");
return name;
} catch (e) {
return '';
}
}

View File

@ -0,0 +1,3 @@
{
"files": ["**/*.js"]
}

View File

@ -0,0 +1,4 @@
{
"zipRegion": "class",
"files": ["**/*.*", "!**/*zipconfig.json"]
}

Binary file not shown.

Binary file not shown.