Building shredMap tooling

README.md

@@ -20,16 +20,8 @@ Angular.io is currently the preview site for Angular 2. This site also includes
 
 ## Development setup with watches and browser reload
  1. cd into root directory `angular.io/`
- 2. install `browser-sync`
+ 2. run `gulp serve-and-sync`
-
+ 3. browser will launch ( on localhost:3000 instead of localhost:9000) and stay refreshed automatically.
-    `npm install -g browser-sync`<br/>
-
-       *or on Windows*<br/>
-
-    `npm install -g browser-sync --msvs_version=2013`
-
- 3. run `gulp serve-and-sync`
- 4. browser will launch ( on localhost:3000 instead of localhost:9000) and stay refreshed automatically.
 
 ## Technology Used
 - Angular 1.x: The production ready version of Angular

public/doc-shredder/doc-shredder.js

@@ -7,15 +7,7 @@ var delPromise =  Q.denodeify(del);
 var Dgeni = require('dgeni');
 var _ = require('lodash');
 
-var createPackage = function(shredOptions) {
+var resolveShredOptions = function(shredOptions) {
-  var shredder = new Dgeni.Package('doc-shredder', [
-    // require('dgeni-packages/base') - doesn't work
-  ]);
-  shredder.options = resolveOptions(shredOptions);
-  return configure(shredder);
-};
-
-var resolveOptions = function(shredOptions) {
   return _.defaults({}, shredOptions, {
     basePath: path.resolve('.'),
     // read files from any subdir under here
@@ -27,9 +19,20 @@ var resolveOptions = function(shredOptions) {
   });
 }
 
+var resolveMapOptions = function(mapOptions) {
+  return _.defaults({}, mapOptions, {
+    basePath: path.resolve('.'),
+    // read files from any subdir under here
+    sourceDir: "docs",
+    destDir: "docs",
+    // whether to include subdirectories when shredding.
+    includeSubdirs: true
+  });
+}
+
 var shred = function(shredOptions) {
   try {
-    var pkg = createPackage(shredOptions);
+    var pkg = createShredPackage(shredOptions);
     var dgeni = new Dgeni([ pkg]);
     return dgeni.generate();
   } catch(x) {
@@ -58,16 +61,139 @@ var shredSingleDir = function(shredOptions, filePath) {
   });
 }
 
+var getShredMap = function(shredMapOptions) {
+  try {
+    var pkg = createShredMapPackage(shredMapOptions);
+    var dgeni = new Dgeni([ pkg]);
+    return dgeni.generate();
+  } catch(x) {
+    console.log(x.stack);
+    throw x;
+  }
+}
+
+
 module.exports = {
   shred: shred,
   shredSingleDir: shredSingleDir,
-  createPackage: createPackage,
+  resolveShredOptions: resolveShredOptions,
-  resolveOptions: resolveOptions
+  getShredMap: getShredMap
 };
 
-function configure(shredder) {
+function createShredPackage(shredOptions) {
-  var options = shredder.options;
+  var pkg = new Dgeni.Package('doc-shredder', [
-  shredder
+    // require('dgeni-packages/base') - doesn't work
+  ]);
+  var options = resolveShredOptions(shredOptions);
+
+  initializePackage(pkg)
+    .factory(require('./fileShredder'))
+    .factory(require('./regionExtractor'))
+    .processor(require('./mdWrapperProcessor'))
+
+    .config(function(readFilesProcessor, fileShredder ) {
+      readFilesProcessor.fileReaders = [ fileShredder];
+    })
+    // default configs - may be overriden
+    .config(function(readFilesProcessor) {
+      // Specify the base path used when resolving relative paths to source and output files
+      readFilesProcessor.basePath = options.basePath;
+
+      // Specify collections of source files that should contain the documentation to extract
+      var extns = ['*.js', '*.html', '*.ts', '*.css' ];
+      var includeFiles = extns.map(function(extn) {
+        if (options.includeSubdirs) {
+          return path.join(options.sourceDir, '**', extn);
+        } else {
+          return path.join(options.sourceDir, extn);
+        }
+      });
+      readFilesProcessor.sourceFiles = [ {
+        // Process all candidate files in `src` and its subfolders ...
+        include: includeFiles,
+        // When calculating the relative path to these files use this as the base path.
+        // So `src/foo/bar.js` will have relative path of `foo/bar.js`
+        basePath: options.sourceDir
+      } ];
+    })
+    .config(function(writeFilesProcessor) {
+      // Specify where the writeFilesProcessor will write our generated doc files
+      writeFilesProcessor.outputFolder  = options.destDir;
+    });
+  return pkg;
+}
+
+var createShredMapPackage = function(mapOptions) {
+  var pkg = new Dgeni.Package('docshred-mapper', [
+    require('dgeni-packages/base'),
+    require('dgeni-packages/nunjucks')
+  ]);
+  var options = resolveMapOptions(mapOptions);
+
+  initializePackage(pkg)
+    .factory(require('./extractPathsReader'))
+    .processor(require('./shredMapProcessor'))
+
+    .config(function(readFilesProcessor, extractPathsReader ) {
+      readFilesProcessor.fileReaders = [ extractPathsReader];
+    })
+    // default configs - may be overriden
+    .config(function(readFilesProcessor) {
+      // Specify the base path used when resolving relative paths to source and output files
+      readFilesProcessor.basePath = options.basePath;
+
+      // Specify collections of source files that should contain the documentation to extract
+      var extns = ['*.jade' ];
+      var includeFiles = extns.map(function(extn) {
+        if (options.includeSubdirs) {
+          return path.join(options.sourceDir, '**', extn);
+        } else {
+          return path.join(options.sourceDir, extn);
+        }
+      });
+      readFilesProcessor.sourceFiles = [ {
+        // Process all candidate files in `src` and its subfolders ...
+        include: includeFiles,
+        // When calculating the relative path to these files use this as the base path.
+        // So `src/foo/bar.js` will have relative path of `foo/bar.js`
+        basePath: options.sourceDir
+      } ];
+    })
+    .config(function(writeFilesProcessor) {
+      // Specify where the writeFilesProcessor will write our generated doc files
+      writeFilesProcessor.outputFolder  = options.destDir;
+    })
+    .config(function(templateFinder) {
+      // Add a folder to search for our own templates to use when rendering docs
+      templateFinder.templateFolders = [ path.resolve(__dirname) ];
+
+      // Specify how to match docs to templates.
+      // In this case we just use the same static template for all docs
+      templateFinder.templatePatterns = [ '${ doc.docType }.template' ];
+    })
+    .config(function(computePathsProcessor, computeIdsProcessor)  {
+      computePathsProcessor.$enabled = false;
+      computeIdsProcessor.$enabled = false;
+      //computePathsProcessor.pathTemplates.push({
+      //  docTypes: ['foo'],
+      //  pathTemplate: '',
+      //  getOutputPath: function () {
+      //  },
+      //});
+      //
+      //computeIdsProcessor.idTemplates.push({
+      //  docTypes: ['foo'],
+      //  getAliases: function (doc) {
+      //    return [doc.id];
+      //  }
+      //});
+    });
+
+  return pkg;
+}
+
+function initializePackage(pkg) {
+  return pkg
     .processor(require('dgeni-packages/base/processors/read-files'))
     .processor(require('dgeni-packages/base/processors/write-files'))
     .factory(require('dgeni-packages/base/services/writefile'))
@@ -87,50 +213,8 @@ function configure(shredder) {
     .processor({ name: 'docs-rendered', $runAfter: ['rendering-docs'] })
     .processor({ name: 'writing-files', $runAfter: ['docs-rendered'] })
     .processor({ name: 'files-written', $runAfter: ['writing-files'] })
-
-    .factory(require('./fileShredder'))
-    .factory(require('./regionExtractor'))
-    .processor(require('./mdWrapperProcessor'))
-
     .config(function(log) {
       // Set logging level
       log.level = 'info';
     })
-
+}
-
-    .config(function(readFilesProcessor, fileShredder ) {
-      readFilesProcessor.fileReaders = [ fileShredder];
-    })
-
-    // default configs - may be overriden
-    .config(function(readFilesProcessor) {
-
-      // Specify the base path used when resolving relative paths to source and output files
-      readFilesProcessor.basePath = options.basePath;
-
-      // Specify collections of source files that should contain the documentation to extract
-      var extns = ['*.js', '*.html', '*.ts', '*.css' ];
-      var includeFiles = extns.map(function(extn) {
-        if (options.includeSubdirs) {
-          return path.join(options.sourceDir, '**', extn);
-        } else {
-          return path.join(options.sourceDir, extn);
-        }
-      });
-      readFilesProcessor.sourceFiles = [
-        {
-          // Process all candidate files in `src` and its subfolders ...
-          include: includeFiles,
-
-          // When calculating the relative path to these files use this as the base path.
-          // So `src/foo/bar.js` will have relative path of `foo/bar.js`
-          basePath: options.sourceDir
-        }
-      ];
-    })
-    .config(function(writeFilesProcessor) {
-      // Specify where the writeFilesProcessor will write our generated doc files
-      writeFilesProcessor.outputFolder  = options.destDir;
-    });
-  return shredder;
-}

public/doc-shredder/extractPathsReader.js

@@ -0,0 +1,34 @@
+/**
+ * @dgService htmlFileShredder
+ * @description
+ */
+
+var path = require('canonical-path');
+
+module.exports = function extractPathsReader(log) {
+  var rx = /\s*\+makeTabs\(\s*["'](.*?)["']\s*,\s*["'](.*?)["'].*?\)/g
+  return {
+    name: 'extractPathsReader',
+
+    getDocs: function (fileInfo) {
+      var content = fileInfo.content;
+      var refPaths = [];
+      var r;
+      while ((r = rx.exec(content)) !== null) {
+        var basePath = r[1];
+        var fileNames = r[2].split(',');
+        fileNames.forEach(function(fn) {
+          refPaths.push(path.join(basePath, fn));
+        })
+      }
+      if (refPaths.length) {
+        return [{
+          refPaths: refPaths
+        }];
+      } else {
+        return [];
+      }
+    }
+  }
+}
+

public/doc-shredder/shredMapProcessor.js

@@ -0,0 +1,32 @@
+/**
+ * dgProcessor shredMapProcessor
+ * @description
+ *
+ */
+module.exports = function shredMapProcessor(log) {
+  return {
+    $runAfter: ['readFilesProcessor'],
+    $runBefore: ['rendering-docs'],
+    $process: function(docs) {
+      var docMaps = []
+      docs.forEach(function(doc) {
+        var docMap = {
+          jadePath: doc.fileInfo.filePath,
+          jadeRelativePath: doc.fileInfo.projectRelativePath,
+          refPaths: doc.refPaths
+        }
+        docMaps.push(docMap);
+      });
+      var newDocs = [{
+        docType: 'xref-doc.html',
+        docMaps: docMaps,
+        outputPath: 'xref-doc.html'
+      }, {
+        docType: 'xref-doc.js',
+        json: JSON.stringify(docMaps),
+        outputPath: 'xref-doc.js'
+      }]
+      return newDocs;
+    }
+  };
+};

public/doc-shredder/test/gulpfile.js

@@ -6,7 +6,7 @@ var watch = require('gulp-watch');
 
 var docShredder = require('../doc-shredder');
 
-var shredOptions =  docShredder.resolveOptions({
+var shredOptions =  docShredder.resolveShredOptions({
   sourceDir: "test_source",
   destDir:  "test_fragments"
 });
@@ -32,6 +32,16 @@ gulp.task('watch', function (cb) {
   });
 });
 
+gulp.task('map', function() {
+  var options = {
+    sourceDir: 'test_jade',
+    destDir: 'test_jade'
+  }
+  return docShredder.getShredMap(options).then(function(x) {
+    var docMaps = x.docMaps;
+  })
+
+});
 
 gulp.task('default', ['shred']);
 

public/doc-shredder/test/report.md

@@ -0,0 +1,6 @@
+# Jade file -> Example file cross references
+
+        Jade file: - C:/xxx/yyy/zzz 
+           References:  
+              - a/b/c
+              - b/d/e

public/doc-shredder/test/test_jade/guides/setupAlt2.jade

@@ -0,0 +1,120 @@
+include ../../../../_includes/_util-fns
+
+.l-main-section
+  :markdown
+    ## Install Angular2
+    There are four steps to create any Angular app:
+
+    1. Create an entry point HTML file where users will start
+    1. Load the Angular library at the top of the file
+    1. Make a root component for your application
+    1. Bootstrap Angular
+
+    You can edit and test out your apps by serving local files with a web server. Follow the steps in the <a href="../quickstart.html">quickstart</a> to get Typescript setup.
+
+    When you're serving local files, edit and save them and start a web server that serves files in that directory.  If you have Python installed, you can run a basic HTTP server from the root of your code directory with:
+
+  pre.prettyprint.lang-bash
+    code python -m SimpleHTTPServer 8000
+
+.callout.is-helpful
+  header Typescript vs ES5
+  :markdown
+    Although we work through the examples in TypeScript, you can also use
+    regular ES5. Click the ES5 link in any code box to see the ES5 JavaScript
+    version. Note that in ES5, you'd want to name your files `.js` rather than
+    `.ts`.
+
+.l-main-section
+  :markdown
+    ## Create an entry point
+    Create an `index.html` file and add the Angular library tags and a `main.ts` file where
+    you'll build your first component.
+
+    In the `<body>`, add an element called `<my-app>` that will be the root of your
+    application.
+    
+    The TypeScript setup includes System.js, a third-party open-source library that adds ES6 module loading functionality to browsers. This step isn't needed for the ES5 version.
+
+  +makeTabs('gettingstarted', 'ts/index.html,js/index.html', 'TypeScript, JavaScript')
+  
+.callout.is-helpful
+  header Don't use code.angularjs.org in a live app
+  :markdown
+    This example serves the Angular library from <a href="http://code.angularjs.org">code.angularjs.org</a>. This is
+    fine for examples, but you'd want to serve it yourself or use a CDN for real deployment.
+
+.l-main-section
+  :markdown
+    ## Set up the starting component
+
+    In `main.ts`, create a class called `AppComponent`, configure it to bind to the
+    `<my-app>` element in `index.html`, and call Angular's `bootstrap()` to kick
+    it all off like this:
+
+  +makeTabs("gettingstarted", "ts/main.ts, js/main.js", "TypeScript, JavaScript")
+
+  .callout.is-helpful
+    header Annotations vs Decorators
+    :markdown
+      If you are transpiling using a tool that translates the `@` symbols to
+      annotations (for example Traceur), you will need to import the annotation versions of
+      Component and View. That can be easily achieved using
+      `import {ComponentAnnotation as Component, ViewAnnotation as View}`.
+
+.l-main-section
+  :markdown
+    ## Run it!
+
+    Open `index.html` through your web server and you should see:
+
+  figure.image-display
+    img(src='/resources/images/examples/setup-example1.png' alt="Example of Todo App")
+
+.l-main-section
+  :markdown
+    ## Explanations
+
+    This basic Angular app contains the structure for any app you'll build.
+
+  .l-sub-section
+    :markdown
+      ### It's all a tree
+
+      You can think of Angular apps as a tree of components.  This root component we've been talking about acts as the top
+      level container for the rest of your application.  You've named this one `AppComponent`, but there's
+      nothing special about the name and you can use whatever makes sense to you.
+
+      The root component's job is to give a location in the `index.html` file where your application will
+      render through its element, in this case `<my-app>`.  There is also nothing special about this
+      element name; you can pick it as you like.
+
+      The root component loads the initial template for the application that will load other components to perform
+      whatever functions your application needs - menu bars, views, forms, etc.  We'll walk through examples of all of
+      these in the following pages.
+
+  .l-sub-section
+    :markdown
+      ### @Component and @View annotations
+
+      A component annotation describes details about the component. An annotation can be identified by its at-sign (`@`).
+
+      The `@Component` annotation defines the HTML tag for the component by specifying the component's CSS selector.
+
+      The `@View` annotation defines the HTML that represents the component. The component you wrote uses an inline template, but you can also have an external template. To use an external template, specify a <code>templateUrl</code> property and give it the path to the HTML file.
+
+  .l-sub-section
+    :markdown
+      ### import vs. window.angular
+
+      The main difference between the ES5 and TypeScript versions is the loading of modules.
+
+      **TypeScript**<br/>
+      TypeScript supports ES6 module loading syntax. ES6 modules allow for modular loading of JavaScript code. Using ES6 modules you can cherry-pick only what you need for your app.
+
+      **Javascript**<br/>
+      In ES5 the script file creates an angular property on the window of the browser. This property contains every piece of Angular core, whether you need it or not.
+
+  +makeTabs('gettingstarted', 'ts/main-import.ts', 'TypeScript')
+
+  +makeExample('gettingstarted/js', 'main-bootstrap.js', 'JavaScript')

public/doc-shredder/test/test_jade/setupAlt.jade

@@ -0,0 +1,120 @@
+include ../../../../_includes/_util-fns
+
+.l-main-section
+  :markdown
+    ## Install Angular2
+    There are four steps to create any Angular app:
+
+    1. Create an entry point HTML file where users will start
+    1. Load the Angular library at the top of the file
+    1. Make a root component for your application
+    1. Bootstrap Angular
+
+    You can edit and test out your apps by serving local files with a web server. Follow the steps in the <a href="../quickstart.html">quickstart</a> to get Typescript setup.
+
+    When you're serving local files, edit and save them and start a web server that serves files in that directory.  If you have Python installed, you can run a basic HTTP server from the root of your code directory with:
+
+  pre.prettyprint.lang-bash
+    code python -m SimpleHTTPServer 8000
+
+.callout.is-helpful
+  header Typescript vs ES5
+  :markdown
+    Although we work through the examples in TypeScript, you can also use
+    regular ES5. Click the ES5 link in any code box to see the ES5 JavaScript
+    version. Note that in ES5, you'd want to name your files `.js` rather than
+    `.ts`.
+
+.l-main-section
+  :markdown
+    ## Create an entry point
+    Create an `index.html` file and add the Angular library tags and a `main.ts` file where
+    you'll build your first component.
+
+    In the `<body>`, add an element called `<my-app>` that will be the root of your
+    application.
+    
+    The TypeScript setup includes System.js, a third-party open-source library that adds ES6 module loading functionality to browsers. This step isn't needed for the ES5 version.
+
+  +makeTabs('gettingstarted', 'ts/index.html,js/index.html', 'TypeScript, JavaScript')
+  
+.callout.is-helpful
+  header Don't use code.angularjs.org in a live app
+  :markdown
+    This example serves the Angular library from <a href="http://code.angularjs.org">code.angularjs.org</a>. This is
+    fine for examples, but you'd want to serve it yourself or use a CDN for real deployment.
+
+.l-main-section
+  :markdown
+    ## Set up the starting component
+
+    In `main.ts`, create a class called `AppComponent`, configure it to bind to the
+    `<my-app>` element in `index.html`, and call Angular's `bootstrap()` to kick
+    it all off like this:
+
+  +makeTabs("gettingstarted", "ts/main.ts, js/main.js", "TypeScript, JavaScript")
+
+  .callout.is-helpful
+    header Annotations vs Decorators
+    :markdown
+      If you are transpiling using a tool that translates the `@` symbols to
+      annotations (for example Traceur), you will need to import the annotation versions of
+      Component and View. That can be easily achieved using
+      `import {ComponentAnnotation as Component, ViewAnnotation as View}`.
+
+.l-main-section
+  :markdown
+    ## Run it!
+
+    Open `index.html` through your web server and you should see:
+
+  figure.image-display
+    img(src='/resources/images/examples/setup-example1.png' alt="Example of Todo App")
+
+.l-main-section
+  :markdown
+    ## Explanations
+
+    This basic Angular app contains the structure for any app you'll build.
+
+  .l-sub-section
+    :markdown
+      ### It's all a tree
+
+      You can think of Angular apps as a tree of components.  This root component we've been talking about acts as the top
+      level container for the rest of your application.  You've named this one `AppComponent`, but there's
+      nothing special about the name and you can use whatever makes sense to you.
+
+      The root component's job is to give a location in the `index.html` file where your application will
+      render through its element, in this case `<my-app>`.  There is also nothing special about this
+      element name; you can pick it as you like.
+
+      The root component loads the initial template for the application that will load other components to perform
+      whatever functions your application needs - menu bars, views, forms, etc.  We'll walk through examples of all of
+      these in the following pages.
+
+  .l-sub-section
+    :markdown
+      ### @Component and @View annotations
+
+      A component annotation describes details about the component. An annotation can be identified by its at-sign (`@`).
+
+      The `@Component` annotation defines the HTML tag for the component by specifying the component's CSS selector.
+
+      The `@View` annotation defines the HTML that represents the component. The component you wrote uses an inline template, but you can also have an external template. To use an external template, specify a <code>templateUrl</code> property and give it the path to the HTML file.
+
+  .l-sub-section
+    :markdown
+      ### import vs. window.angular
+
+      The main difference between the ES5 and TypeScript versions is the loading of modules.
+
+      **TypeScript**<br/>
+      TypeScript supports ES6 module loading syntax. ES6 modules allow for modular loading of JavaScript code. Using ES6 modules you can cherry-pick only what you need for your app.
+
+      **Javascript**<br/>
+      In ES5 the script file creates an angular property on the window of the browser. This property contains every piece of Angular core, whether you need it or not.
+
+  +makeTabs('gettingstarted', 'ts/main-import.ts', 'TypeScript')
+
+  +makeExample('gettingstarted/js', 'main-bootstrap.js', 'JavaScript')

public/doc-shredder/test/test_jade/xref-doc.html

@@ -0,0 +1,28 @@
+<h1> Jade document --> example Cross Reference Report
+
+<ol>
+    <li>
+        Jade file: <strong>test_jade/guides/setupAlt2.jade</strong>
+        <p>Reference paths</p>
+        <ul>
+            <li>Path: gettingstarted/ts/index.html</li>
+            <li>Path: gettingstarted/js/index.html</li>
+            <li>Path: gettingstarted/ts/main.ts</li>
+            <li>Path: gettingstarted/ js/main.js</li>
+            <li>Path: gettingstarted/ts/main-import.ts</li>
+        </ul>
+        <br>
+    </li>
+    <li>
+        Jade file: <strong>test_jade/setupAlt.jade</strong>
+        <p>Reference paths</p>
+        <ul>
+            <li>Path: gettingstarted/ts/index.html</li>
+            <li>Path: gettingstarted/js/index.html</li>
+            <li>Path: gettingstarted/ts/main.ts</li>
+            <li>Path: gettingstarted/ js/main.js</li>
+            <li>Path: gettingstarted/ts/main-import.ts</li>
+        </ul>
+        <br>
+    </li>
+</ol>

public/doc-shredder/test/test_jade/xref-doc.js

@@ -0,0 +1 @@
+[{"jadePath":"c:/GitHub/angular.io.ideablade/public/doc-shredder/test/test_jade/guides/setupAlt2.jade","jadeRelativePath":"test_jade/guides/setupAlt2.jade","refPaths":["gettingstarted/ts/index.html","gettingstarted/js/index.html","gettingstarted/ts/main.ts","gettingstarted/ js/main.js","gettingstarted/ts/main-import.ts"]},{"jadePath":"c:/GitHub/angular.io.ideablade/public/doc-shredder/test/test_jade/setupAlt.jade","jadeRelativePath":"test_jade/setupAlt.jade","refPaths":["gettingstarted/ts/index.html","gettingstarted/js/index.html","gettingstarted/ts/main.ts","gettingstarted/ js/main.js","gettingstarted/ts/main-import.ts"]}]

public/doc-shredder/xref-doc.html.template

@@ -0,0 +1,16 @@
+<h1> Jade document --> example Cross Reference Report
+
+<ol>
+    {%- for docMap in doc.docMaps %}
+    <li>
+        Jade file: <strong>{{ docMap.jadeRelativePath }}</strong>
+        <p>Reference paths</p>
+        <ul>
+            {%- for refPath in docMap.refPaths %}
+            <li>Path: {{ refPath }}</li>
+            {%- endfor %}
+        </ul>
+        <br>
+    </li>
+    {%- endfor %}
+</ol>

public/doc-shredder/xref-doc.js.template

@@ -0,0 +1 @@
+{{ doc.json }}