diff --git a/.gitignore b/.gitignore index 156b58bb6d..98d8515850 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ pubspec.lock **/ts/latest/api **/docs/_fragments public/docs/xref-*.* +_zip-output diff --git a/gulpfile.js b/gulpfile.js index e3d889b12e..425508cff5 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -23,18 +23,22 @@ var prompt = require('prompt'); var docShredder = require('./public/doc-shredder/doc-shredder'); +var exampleZipper = require('./public/_example-zipper/exampleZipper'); var _devguideShredOptions = { examplesDir: './public/docs/_examples', - fragmentsDir: './public/docs/_fragments' + fragmentsDir: './public/docs/_fragments', + zipDir: './public/resources/zips' }; var _apiShredOptions = { examplesDir: '../angular/modules/angular2/examples', - fragmentsDir: './public/docs/_fragments/_api' + fragmentsDir: './public/docs/_fragments/_api', + zipDir: './public/resources/zips/api' }; + var _excludePatterns = ['**/node_modules/**', '**/typings/**', '**/packages/**']; var _excludeMatchers = _excludePatterns.map(function(excludePattern){ @@ -85,7 +89,7 @@ gulp.task('build-and-serve', ['build-docs'], function (cb) { }); }); -gulp.task('build-docs', ['_shred-devguide-examples', 'build-api-docs'], function() { +gulp.task('build-docs', ['_shred-devguide-examples', 'build-api-docs', '_zip-examples'], function() { return buildShredMaps(true); }); @@ -122,6 +126,11 @@ gulp.task('_build-shred-maps', function() { return build-shred-maps(true); }); +gulp.task('_zip-examples', function() { + exampleZipper.zipExamples(_devguideShredOptions.examplesDir, _devguideShredOptions.zipDir); + exampleZipper.zipExamples(_apiShredOptions.examplesDir, _apiShredOptions.zipDir); +}); + gulp.task('git-changed-examples', ['_shred-devguide-examples'], function(){ var after, sha, messageSuffix; if (argv.after) { diff --git a/package.json b/package.json index 1caa10c9b1..7128b1ac4a 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,8 @@ "url": "" }, "devDependencies": { + "archiver": "^0.16.0", + "assert-plus": "^0.1.5", "browser-sync": "^2.9.3", "canonical-path": "0.0.2", "del": "^1.2.0", @@ -50,6 +52,7 @@ "lodash": "^3.10.1", "marked": "^0.3.5", "minimatch": "^2.0.10", + "mkdirp": "^0.5.1", "node-html-encoder": "0.0.2", "nodegit": "0.5.0", "path": "^0.11.14", diff --git a/public/_example-zipper/Ideas.md b/public/_example-zipper/Ideas.md new file mode 100644 index 0000000000..ee41ef8094 --- /dev/null +++ b/public/_example-zipper/Ideas.md @@ -0,0 +1,15 @@ +Summary: + + 1) if we discover a 'zipconfig.json' file or an 'xxx.zipconfig.json' file with the following structure + + { + "zipRegion": "zip", + "files": [ "foo.js", "**/**/.css", "!xxx.css"] + } + + where "zipRegion" is optional. + + Then we zip up all of the files specified, cleaning and removing extra doc tags. If the specified 'zipRegion' + is discovered in any file then we only zip that region. Otherwise we zip the entire file. + + \ No newline at end of file diff --git a/public/_example-zipper/exampleZipper.js b/public/_example-zipper/exampleZipper.js new file mode 100644 index 0000000000..3bdbdd4f70 --- /dev/null +++ b/public/_example-zipper/exampleZipper.js @@ -0,0 +1,98 @@ +// Canonical path provides a consistent path (i.e. always forward slashes) across different OSes +var path = require('canonical-path'); +var Q = require('q'); +var _ = require('lodash'); +var jsonfile = require('jsonfile'); +var assert = require('assert-plus'); +// adm-zip does not work properly on Windows +// var Zip = require('adm-zip'); +var archiver = require('archiver'); + +var fs = require('fs'); +var mkdirp = require('mkdirp'); + +var globule = require('globule'); + +var regionExtractor = require('../doc-shredder/regionExtractor'); + + +// globs is an array of globs +function zipExamples(sourceDirName, outputDirName) { + var gpath = path.join(sourceDirName, '**/*zipconfig.json'); + var configFileNames = globule.find([gpath]); + configFileNames.forEach(function(configFileName) { + zipExample(configFileName, sourceDirName, outputDirName); + }); +} + +function zipExample(configFileName, sourceDirName, outputDirName) { + var json = jsonfile.readFileSync(configFileName); + var fileGlobs = json.files; + var zipRegionName = json.zipRegion; + + // assert that fileGlobs is an array + assert.arrayOfString(fileGlobs, 'files property should be an array of strings'); + + // backup two from the relative configFileName + var relativeDirName = path.dirname(path.dirname(path.relative(sourceDirName, configFileName))); + var configDirName = path.dirname(configFileName); + // use the dir name of the folder containing the config file as the base file name of the zip file + var baseFileName = path.basename(configDirName); + // check if there is a prefix name before zipconfig.json + if (configFileName.indexOf('.zipconfig.json') >= 0) { + // if so use it as a suffix to the baseFileName + var extraName = path.basename(configFileName, '.zipconfig.json'); + baseFileName = baseFileName + "." + extraName; + } + + var outputFileName = path.join(outputDirName, relativeDirName, baseFileName + ".zip"); + var fileNames = globule.find(fileGlobs, { srcBase: configDirName, prefixBase: configDirName }); + + var zip = createZipArchive(outputFileName); + fileNames.forEach(function(fileName) { + var relativePath = path.relative(configDirName, fileName); + var content = fs.readFileSync(fileName, 'utf8'); + var extn = path.extname(fileName).substr(1); + // if we don't need to clean up the file then we can do the following. + // zip.append(fs.createReadStream(fileName), { name: relativePath }); + var output; + // if no zipRegion or zipRegion is not in content then just clean the content + if (zipRegionName != null) { + output = regionExtractor.getRegionDoc(content, extn, zipRegionName); + } + if (!output) { + output = regionExtractor.removeDocTags(content, extn); + } + + zip.append(output, { name: relativePath } ) + }); + + zip.finalize(); +} + +function createZipArchive(zipFileName) { + var dirName = path.dirname(zipFileName); + // insure that the folder exists. + if (!fs.existsSync(dirName)) { + mkdirp.sync(dirName); + } + var output = fs.createWriteStream(zipFileName); + var archive = archiver('zip'); + + output.on('close', function () { + console.log('zip created: ' + zipFileName + ' (' + archive.pointer() + ' total bytes)'); + }); + + archive.on('error', function (err) { + throw err; + }); + + archive.pipe(output); + return archive; +} + + +module.exports = { + zipExamples: zipExamples, + zipExample: zipExample +}; \ No newline at end of file diff --git a/public/_example-zipper/test/gulpfile.js b/public/_example-zipper/test/gulpfile.js new file mode 100644 index 0000000000..aac02c468b --- /dev/null +++ b/public/_example-zipper/test/gulpfile.js @@ -0,0 +1,26 @@ +var gulp = require('gulp'); +var path = require('canonical-path'); +var del = require('del'); +var taskListing = require('gulp-task-listing'); + +var exampleZipper = require('../exampleZipper'); + +var _outputFolder = '_zip-output'; + +gulp.task('help', taskListing); + +gulp.task('zipExamples', ['clean'], function() { + return exampleZipper.zipExamples("../../docs/_examples", _outputFolder); +}); + +gulp.task('clean', function (cb) { + var cleanPath = path.join(_outputFolder, '**/*.*'); + del([ cleanPath, '!**/*.ovr.*'], function (err, paths) { + // console.log('Deleted files/folders:\n', paths.join('\n')); + cb(); + }); +}); + + +gulp.task('default', taskListing); + diff --git a/public/doc-shredder/doc-shredder.js b/public/doc-shredder/doc-shredder.js index deb0bccfa0..22eabc1fed 100644 --- a/public/doc-shredder/doc-shredder.js +++ b/public/doc-shredder/doc-shredder.js @@ -63,7 +63,6 @@ function createShredPackage(shredOptions) { initializePackage(pkg) .factory(require('./fileShredder')) - .factory(require('./regionExtractor')) .processor(require('./mdWrapperProcessor')) .config(function(readFilesProcessor, fileShredder ) { diff --git a/public/doc-shredder/fileShredder.js b/public/doc-shredder/fileShredder.js index 0869689725..4231ef0799 100644 --- a/public/doc-shredder/fileShredder.js +++ b/public/doc-shredder/fileShredder.js @@ -1,57 +1,21 @@ +var regionExtractor = require('./regionExtractor'); +var buildRegionDocs = regionExtractor.buildRegionDocs; + /** * @dgService htmlFileShredder * @description */ -module.exports = function fileShredder(log, regionExtractor) { +module.exports = function fileShredder(log ) { return { name: 'fileShredder', getDocs: function (fileInfo) { - var commentInfo; - switch (fileInfo.extension) { - case 'ts': - case 'js': - case 'dart': - commentInfo = { - prefix: '//', - blockPattern: '/* {tag} */' - }; - //commentMarkers = ['//']; - break; - case 'html': - commentInfo = { - prefix: '' - }; - // commentMarkers = [' for html - // or // #docregion foo for js/ts - // 2) In back: a comment marker followed by '#enddocregion' - // Regions can be nested and any regions not 'closed' are automatically closed at the end of the doc. - - // empty enddocregion always closes last region started. - // enddocregions with names that do no match start region tags get ignored. - - return function(content, commentInfo) { - - var lines = result = content.split(/\r?\n/); - - var docStack = []; // items will be both popped and removed from the middle - var docMap = {}; - var doc; - var regionNames; - var docPlaster = '. . .'; - lines.forEach(function(line, ix) { - if (isCommentLine(line, commentInfo.prefix)) { - if (hasRegionTag(line)) { - lines[ix] = nullLine; - regionNames = getRegionNames(line); - regionNames.forEach(function(rn) { - doc = docMap[rn]; - if (!doc) { - // regionName may be '' - doc = {regionName: rn, ranges: [ { startIx: ix} ] }; - docMap[rn] = doc; - } else { - // only add a new range if prev range is closed - var lastRange = doc.ranges[doc.ranges.length-1]; - if (lastRange.endIx) { - doc.ranges.push({startIx: ix}); - } - } - docStack.push(doc); - }); - - } else if (hasEndRegionTag(line)) { - lines[ix] = nullLine; - regionNames = getEndRegionNames(line); - regionNames.forEach(function(rn) { - // handle endregions with no name specially. - // They operate on the last region created. - if (rn.length == 0) { - if (docStack.length) { - // update last item on the stack - doc = docStack.pop(); - doc.ranges[doc.ranges.length - 1].endIx = ix; - } - } else { - doc = docMap[rn]; - // ignore endregion if name is specified but not found. - if (doc) { - doc.ranges[doc.ranges.length - 1].endIx = ix; - // remove doc from stack - _.remove(docStack, function (item) { - return item.regionName === rn; - }); - } - } - }); - } else if (hasDocPlasterTag(line)) { - line[ix] = nullLine; - docPlaster = getDocPlaster(line); - } - } - }); - - var docs = _.values(docMap); - var plasterComment = docPlaster && commentInfo.blockPattern.replace('{tag}', docPlaster); - docs.forEach(function(doc) { - var content; - var fragLines = []; - doc.ranges.forEach(function (range) { - var subLines; - if (range.endIx) { - subLines = lines.slice(range.startIx + 1, range.endIx); - } else { - subLines = lines.slice(range.startIx + 1); - } - if (plasterComment && fragLines.length) { - // pad is the padding on the previous line - var pad = fragLines[fragLines.length - 1].match(/(\s*)/)[0]; - fragLines.push(pad + plasterComment); - } - fragLines = fragLines.concat(subLines); - }); - fragLines = trimLeftIndent(fragLines); - content = fragLines.join('\n'); - // eliminate all #docregion lines - content = content.replace(nullLinePattern, ''); - if (content.substr(-3) === nullLine) { - content = content.substr(0, content.length-3); - } - doc.content = content; - }); - return docs; - } +var nullLine = '###'; +var nullLinePattern = new RegExp(nullLine + '\n', 'g'); +module.exports = { + buildRegionDocs: buildRegionDocs, + getRegionDoc: getRegionDoc, + removeDocTags: removeDocTags }; -function trimLeftIndent(lines) { - var minIx = 100; - var ok = lines.every(function(line) { - // var ix = line.search(/\S/); - var ix = line.search(/[^ ]/); - if (ix === 0) return false; - if (ix === -1) return true; - if (ix > 0) { - minIx = Math.min(minIx, ix); - } - return true; - }); - if ( (!ok) || minIx === 100) return lines; +// split out each fragment in {content} into a separate doc +// a fragment is a section of text surrounded by +// 1) In front: a comment marker followed by '#docregion' followed by an optional region name. For example: +// <-- #docregion foo --> for html +// or // #docregion foo for js/ts +// 2) In back: a comment marker followed by '#enddocregion' +// Regions can be nested and any regions not 'closed' are automatically closed at the end of the doc. - var result = lines.map(function(line) { - if (line.length > minIx) { - return line.substr(minIx); - } else { - // this can happen if line is all blanks and shorter than mixIx - return line; +// empty enddocregion always closes last region started. +// enddocregions with names that do no match start region tags get ignored. +function buildRegionDocs(content, extn) { + var commentInfo = getCommentInfo(extn); + if (!commentInfo) { + return [ { content: content } ]; + } + + var lines = result = content.split(/\r?\n/); + + var docStack = []; // items will be both popped and removed from the middle + var docMap = {}; + var docPlaster = '. . .'; + var doc; + var regionNames; + lines.forEach(function(line, ix) { + if (isCommentLine(line, commentInfo.prefix)) { + if (hasRegionTag(line)) { + lines[ix] = nullLine; + regionNames = getRegionNames(line); + regionNames.forEach(function(rn) { + doc = docMap[rn]; + if (!doc) { + // regionName may be '' + doc = {regionName: rn, ranges: [ { startIx: ix} ] }; + docMap[rn] = doc; + } else { + // only add a new range if prev range is closed + var lastRange = doc.ranges[doc.ranges.length-1]; + if (lastRange.endIx) { + doc.ranges.push({startIx: ix}); + } + } + docStack.push(doc); + }); + + } else if (hasEndRegionTag(line)) { + lines[ix] = nullLine; + regionNames = getEndRegionNames(line); + regionNames.forEach(function(rn) { + // handle endregions with no name specially. + // They operate on the last region created. + if (rn.length == 0) { + if (docStack.length) { + // update last item on the stack + doc = docStack.pop(); + doc.ranges[doc.ranges.length - 1].endIx = ix; + } + } else { + doc = docMap[rn]; + // ignore endregion if name is specified but not found. + if (doc) { + doc.ranges[doc.ranges.length - 1].endIx = ix; + // remove doc from stack + _.remove(docStack, function (item) { + return item.regionName === rn; + }); + } + } + }); + } else if (hasDocPlasterTag(line)) { + line[ix] = nullLine; + docPlaster = getDocPlaster(line); + } } }); + + var docs = _.values(docMap); + var plasterComment = docPlaster && commentInfo.plasterPattern.replace('{tag}', docPlaster); + docs = reattachDocs(docs, lines, plasterComment); + return docs; +} + +function getRegionDoc(content, extn, regionName) { + var docs = buildRegionDocs(content, extn); + var doc = _.find(docs, function (doc) { + return doc.regionName === regionName; + }); + return doc && doc.content; +} + +function removeDocTags(content, extn) { + var commentInfo = getCommentInfo(extn); + if (commentInfo == null) { + return content; + } + var lines = result = content.split(/\r?\n/); + + lines.forEach(function(line, ix) { + if (isCommentLine(line, commentInfo.prefix)) { + if (hasDocTag(line)) { + lines[ix] = nullLine; + } + } + }); + var result = joinLines(lines); return result; } +function reattachDocs(docs, lines, plasterComment) { + + docs.forEach(function(doc) { + var content; + var fragLines = []; + doc.ranges.forEach(function (range) { + var subLines; + if (range.endIx) { + subLines = lines.slice(range.startIx + 1, range.endIx); + } else { + subLines = lines.slice(range.startIx + 1); + } + if (plasterComment && fragLines.length) { + // pad is the padding on the previous line + var pad = fragLines[fragLines.length - 1].match(/(\s*)/)[0]; + fragLines.push(pad + plasterComment); + } + fragLines = fragLines.concat(subLines); + }); + fragLines = trimLeftIndent(fragLines); + doc.content = joinLines(fragLines); + }); + return docs; +} + + + +function getCommentInfo(extension) { + var commentInfo; + switch (extension) { + case 'ts': + case 'js': + case 'dart': + commentInfo = { + prefix: '//', + plasterPattern: '/* {tag} */' + }; + break; + case 'html': + commentInfo = { + prefix: '' + }; + break; + case 'css': + commentInfo = { + prefix: '/*', + plasterPattern: '/* {tag} */' + }; + break; + case 'json': + return null; + case 'yaml': + commentInfo = { + prefix: '#', + plasterPattern: '# {tag} ' + }; + break; + default: + return null; + } + return commentInfo; +} + function isCommentLine(line, commentPrefix) { return line.trim().indexOf(commentPrefix) == 0; } +function hasDocTag(line) { + return hasRegionTag(line) || hasEndRegionTag(line) || hasDocPlasterTag(line); +} + function hasRegionTag(line) { return line.indexOf("#docregion") >= 0; } @@ -181,3 +236,38 @@ function extractRegionNames(line, rx) { return ['']; } } + +function trimLeftIndent(lines) { + var minIx = 100; + var ok = lines.every(function(line) { + // var ix = line.search(/\S/); + var ix = line.search(/[^ ]/); + if (ix === 0) return false; + if (ix === -1) return true; + if (ix > 0) { + minIx = Math.min(minIx, ix); + } + return true; + }); + if ( (!ok) || minIx === 100) return lines; + + var result = lines.map(function(line) { + if (line.length > minIx) { + return line.substr(minIx); + } else { + // this can happen if line is all blanks and shorter than mixIx + return line; + } + }); + return result; +} + +function joinLines(lines) { + var content = lines.join('\n'); + // eliminate all #docregion lines + content = content.replace(nullLinePattern, ''); + if (content.substr(-3) === nullLine) { + content = content.substr(0, content.length - 3); + } + return content; +} \ No newline at end of file diff --git a/public/doc-shredder/regionExtractor.old.js b/public/doc-shredder/regionExtractor.old.js deleted file mode 100644 index e01bc718e0..0000000000 --- a/public/doc-shredder/regionExtractor.old.js +++ /dev/null @@ -1,106 +0,0 @@ -module.exports = function regionExtractor() { - - var nullLine = '###'; - var nullLinePattern = new RegExp(nullLine + '\n', 'g'); - - // split out each fragment in {content} into a separate doc - // a fragment is a section of text surrounded by - // 1) In front: a comment marker followed by '#docregion' followed by an optional region name. For example: - // <-- #docregion foo --> for html - // or // #docregion foo for js/ts - // 2) In back: a comment marker followed by '#enddocregion' - // Regions can be nested and any regions not 'closed' are automatically closed at the end of the doc. - return function(content, commentPrefixes) { - - var lines = result = content.split(/\r?\n/); - var docs = []; - var docStack = []; - var doc = null; - - lines.forEach(function(line, ix) { - if (isCommentLine(line, commentPrefixes)) { - if (hasRegionTag(line)) { - if (doc) docStack.push(doc); - doc = {startIx: ix, regionName: getRegionName(line)}; - lines[ix] = nullLine; - docs.push(doc); - } else if (hasEndRegionTag(line)) { - if (doc) { - lines[ix] = nullLine; - doc.endIx = ix; - doc = docStack.pop(); - } - } - } - }); - - docs.forEach(function(doc) { - var fragLines, content; - if (doc.endIx) { - fragLines = lines.slice(doc.startIx + 1, doc.endIx); - } else { - fragLines = lines.slice(doc.startIx + 1); - } - fragLines = trimLeftIndent(fragLines); - content = fragLines.join('\n'); - // eliminate all #docregion lines - content = content.replace(nullLinePattern, ''); - if (content.substr(-3) === nullLine) { - content = content.substr(0, content.length-3); - } - doc.content = content; - }); - return docs; - } - -}; - -function trimLeftIndent(lines) { - var minIx = 100; - var ok = lines.every(function(line) { - // var ix = line.search(/\S/); - var ix = line.search(/[^ ]/); - if (ix === 0) return false; - if (ix === -1) return true; - if (ix > 0) { - minIx = Math.min(minIx, ix); - } - return true; - }); - if ( (!ok) || minIx === 100) return lines; - - var result = lines.map(function(line) { - if (line.length > minIx) { - return line.substr(minIx); - } else { - // this can happen if line is all blanks and shorter than mixIx - return line; - } - }); - return result; -} - -function isCommentLine(line, commentPrefixes) { - return commentPrefixes.some(function(prefix) { - return line.trim().indexOf(prefix) == 0; - }); -} - -function hasRegionTag(line) { - return line.indexOf("#docregion") >= 0; -} - -function hasEndRegionTag(line) { - return line.indexOf("#enddocregion") >= 0; -} - -function getRegionName(line) { - try { - var name = line.match(/#docregion\s*(\S*).*/)[1]; - // Hack for html regions that look like or */ - name = name.replace("-->","").replace('\*\/',""); - return name; - } catch (e) { - return ''; - } -} diff --git a/public/docs/_examples/styleguide/jsonly.zipconfig.json b/public/docs/_examples/styleguide/jsonly.zipconfig.json new file mode 100644 index 0000000000..118c776bdc --- /dev/null +++ b/public/docs/_examples/styleguide/jsonly.zipconfig.json @@ -0,0 +1,3 @@ +{ + "files": ["**/*.js"] +} \ No newline at end of file diff --git a/public/docs/_examples/styleguide/zipconfig.json b/public/docs/_examples/styleguide/zipconfig.json new file mode 100644 index 0000000000..74ca947f2b --- /dev/null +++ b/public/docs/_examples/styleguide/zipconfig.json @@ -0,0 +1,4 @@ +{ + "zipRegion": "class", + "files": ["**/*.*", "!**/*zipconfig.json"] +} \ No newline at end of file diff --git a/public/resources/zips/styleguide.jsonly.zip b/public/resources/zips/styleguide.jsonly.zip new file mode 100644 index 0000000000..1e12e8dd58 Binary files /dev/null and b/public/resources/zips/styleguide.jsonly.zip differ diff --git a/public/resources/zips/styleguide.zip b/public/resources/zips/styleguide.zip new file mode 100644 index 0000000000..4d749968f5 Binary files /dev/null and b/public/resources/zips/styleguide.zip differ