build(aio): implement prerendering

The current implementation is based on @igorminar's [angular-io-v42][1]. It is
using Protractor to request all docs URLs, let them fallback to `/index.html`
and save the rendered page.

[1]: https://github.com/IgorMinar/angular-io-v42/tree/05508ab3/tools/prerenderer

Fixes #15104

aio/.gitignore

@@ -7,5 +7,6 @@ yarn-error.log
 
 # Ignore generated content
 /dist
+/tmp
 /src/content/docs
 /.sass-cache

aio/aio-builds-setup/dockerbuild/nginx/aio-builds.conf

@@ -35,7 +35,7 @@ server {
   }
 
   location / {
-    try_files $uri $uri/ /index.html =404;
+    try_files $uri /content/docs-prerendered/$uri.html $uri/ =404;
   }
 }
 

aio/firebase.json

@@ -4,11 +4,10 @@
   },
   "hosting": {
     "public": "dist",
-    "cleanUrls": true,
     "rewrites": [
       {
-        "source": "**",
+        "source": "/:path*",
-        "destination": "/index.html"
+        "destination": "/content/docs-prerendered/:path*.html"
       }
     ]
   }

aio/package.json

@@ -14,13 +14,20 @@
     "lint": "yarn check-env && ng lint",
     "pree2e": "yarn ~~update-webdriver",
     "e2e": "yarn check-env && ng e2e --no-webdriver-update",
+    "predeploy-preview": "yarn prerender",
     "deploy-preview": "scripts/deploy-preview.sh",
+    "predeploy-staging": "yarn prerender",
     "deploy-staging": "firebase use staging --token \"$FIREBASE_TOKEN\" && yarn ~~deploy",
     "check-env": "node ../tools/check-environment.js",
     "docs": "dgeni ./transforms/angular.io-package",
     "docs-test": "node ../dist/tools/cjs-jasmine/index-tools ../../transforms/**/*.spec.js",
+    "preprerender": "yarn build",
+    "prerender": "concurrently --kill-others --raw --success first \"yarn ~~prerender-serve\" \"yarn ~~prerender\"",
+    "postprerender": "node tools/prerender/copy-to-dist",
     "~~update-webdriver": "webdriver-manager update --standalone false --gecko false",
-    "pre~~deploy": "yarn build",
+    "pre~~prerender": "yarn ~~update-webdriver",
+    "~~prerender": "node tools/prerender/create-specs && protractor tools/prerender/protractor.conf.js",
+    "~~prerender-serve": "node tools/prerender/serve",
     "~~deploy": "firebase deploy --message \"Commit: $TRAVIS_COMMIT\" --non-interactive --token \"$FIREBASE_TOKEN\""
   },
   "private": true,
@@ -49,6 +56,7 @@
     "@types/node": "~6.0.60",
     "canonical-path": "^0.0.2",
     "codelyzer": "~2.0.0-beta.4",
+    "concurrently": "^3.4.0",
     "dgeni": "^0.4.7",
     "dgeni-packages": "^0.16.8",
     "entities": "^1.1.1",
@@ -65,6 +73,7 @@
     "lodash": "^4.17.4",
     "protractor": "~5.1.0",
     "rho": "^0.3.0",
+    "shelljs": "^0.7.7",
     "ts-node": "~2.0.0",
     "tslint": "~4.4.2",
     "typescript": "2.1.6"

aio/protractor.conf.js

@@ -1,5 +1,6 @@
 // Protractor configuration file, see link for more information
 // https://github.com/angular/protractor/blob/master/lib/config.ts
+'use strict';
 
 /*global jasmine */
 const { SpecReporter } = require('jasmine-spec-reporter');

aio/scripts/deploy-preview.sh

@@ -11,7 +11,7 @@ UPLOAD_URL=$AIO_BUILDS_HOST/create-build/$TRAVIS_PULL_REQUEST/$TRAVIS_PULL_REQUE
 
 cd "`dirname $0`/.."
 
-yarn run build
+# Assumes the build step has already run
 tar --create --gzip --directory "$INPUT_DIR" --file "$OUTPUT_FILE" .
 
 exec 3>&1

aio/tools/prerender/constants.js

@@ -0,0 +1,30 @@
+'use strict';
+
+// Imports
+const path = require('path');
+
+// Constants
+const BROWSER_INSTANCES = 7;
+
+const PORT = 4201;
+const BASE_URL = `http://localhost:${PORT}`;
+
+const ROOT_DIR = path.join(__dirname, '../..');
+const DIST_DIR = path.join(ROOT_DIR, 'dist');
+const CONTENT_DIR = path.join(DIST_DIR, 'content');
+const INPUT_DIR = path.join(CONTENT_DIR, 'docs');
+const TMP_SPECS_DIR = path.join(ROOT_DIR, 'tmp/docs-prerender-specs');
+const TMP_OUTPUT_DIR = path.join(ROOT_DIR, 'tmp/docs-prerendered');
+
+// Exports
+module.exports = {
+  BASE_URL,
+  BROWSER_INSTANCES,
+  CONTENT_DIR,
+  DIST_DIR,
+  INPUT_DIR,
+  PORT,
+  ROOT_DIR,
+  TMP_OUTPUT_DIR,
+  TMP_SPECS_DIR
+};

aio/tools/prerender/copy-to-dist.js

@@ -0,0 +1,10 @@
+'use strict';
+
+// Imports
+const sh = require('shelljs');
+const { CONTENT_DIR, TMP_OUTPUT_DIR } = require('./constants');
+
+sh.config.fatal = true;
+
+// Run
+sh.cp('-r', TMP_OUTPUT_DIR, CONTENT_DIR);

aio/tools/prerender/create-specs.js

@@ -0,0 +1,89 @@
+'use strict';
+
+// Imports
+const fs = require('fs');
+const path = require('path');
+const sh = require('shelljs');
+const { BASE_URL, BROWSER_INSTANCES, INPUT_DIR, PORT, TMP_OUTPUT_DIR, TMP_SPECS_DIR } = require('./constants');
+
+sh.config.fatal = true;
+
+// Helpers
+const chunkArray = (items, numChunks) => {
+  numChunks = Math.min(numChunks, items.length);
+  const itemsPerChunk = Math.ceil(items.length / numChunks);
+  const chunks = new Array(numChunks);
+
+  console.log(`Chunking ${items.length} items into ${numChunks} chunks.`);
+
+  for (let i = 0; i < numChunks; i++) {
+    chunks[i] = items.slice(i * itemsPerChunk, (i + 1) * itemsPerChunk);
+  }
+
+  return chunks;
+};
+
+const getAllFiles = rootDir => fs.readdirSync(rootDir).reduce((files, file) => {
+  const absolutePath = path.join(rootDir, file);
+  const isFile = fs.lstatSync(absolutePath).isFile();
+
+  return files.concat(isFile ? absolutePath : getAllFiles(absolutePath));
+}, []);
+
+const getAllUrls = rootDir => getAllFiles(rootDir).
+  filter(absolutePath => path.extname(absolutePath) === '.json').
+  map(absolutePath => absolutePath.slice(0, -5)).
+  map(absolutePath => path.relative(INPUT_DIR, absolutePath)).
+  map(relativePath => `${BASE_URL}/${relativePath}`);
+
+const getTestForChunk = (chunk, idx) => `
+  'use strict';
+
+  const fs = require('fs');
+  const path = require('path');
+  const protractor = require('protractor');
+  const sh = require('shelljs');
+  const url = require('url');
+
+  const browser = protractor.browser;
+  sh.config.fatal = true;
+
+  describe('chunk ${idx}', () => ${JSON.stringify(chunk)}.forEach(urlToPage => {
+    const parsedUrl = url.parse(urlToPage);
+
+    it(\`should render \${parsedUrl.path}\`, done => {
+      browser.get(urlToPage);
+      browser.getPageSource()
+        .then(source => {
+          if (/document not found/i.test(source) && !/file-not-found/i.test(urlToPage)) {
+            return Promise.reject(\`404 for \${urlToPage}\`);
+          }
+
+          const relativeFilePath = parsedUrl.path.replace(/\\/$/, '/index').replace(/^\\//, '') + '.html';
+          const absoluteFilePath = path.resolve('${TMP_OUTPUT_DIR}', relativeFilePath);
+          const absoluteDirPath = path.dirname(absoluteFilePath);
+
+          console.log(\`Writing to \${absoluteFilePath}...\`);
+
+          sh.mkdir('-p', absoluteDirPath);
+          fs.writeFileSync(absoluteFilePath, source);
+        })
+        .then(done, done.fail);
+    });
+  }));
+`;
+
+// Run
+const docsUrls = getAllUrls(INPUT_DIR);
+const chunked = chunkArray(docsUrls, BROWSER_INSTANCES);
+
+sh.rm('-rf', TMP_OUTPUT_DIR);
+sh.rm('-rf', TMP_SPECS_DIR);
+sh.mkdir('-p', TMP_SPECS_DIR);
+
+chunked.forEach((chunk, idx) => {
+  const outputFile = path.join(TMP_SPECS_DIR, `chunk${idx}.spec.js`);
+  const testContent = getTestForChunk(chunk, idx);
+
+  fs.writeFileSync(outputFile, testContent);
+});

aio/tools/prerender/protractor.conf.js

@@ -0,0 +1,35 @@
+// Protractor configuration file, see link for more information
+// https://github.com/angular/protractor/blob/master/lib/config.ts
+'use strict';
+
+/*global jasmine */
+const { SpecReporter } = require('jasmine-spec-reporter');
+const path = require('path');
+const { BASE_URL, BROWSER_INSTANCES, TMP_SPECS_DIR } = require('./constants');
+
+exports.config = {
+  allScriptsTimeout: 11000,
+  specs: [
+    path.join(TMP_SPECS_DIR, 'chunk*.spec.js')
+  ],
+  capabilities: {
+    browserName: 'chrome',
+    shardTestFiles: true,
+    maxInstances: BROWSER_INSTANCES,
+    // For Travis
+    chromeOptions: {
+      binary: process.env.CHROME_BIN
+    }
+  },
+  directConnect: true,
+  baseUrl: BASE_URL,
+  framework: 'jasmine',
+  jasmineNodeOpts: {
+    showColors: true,
+    defaultTimeoutInterval: 30000,
+    print: function() {}
+  },
+  onPrepare() {
+    jasmine.getEnv().addReporter(new SpecReporter({spec: {displayStacktrace: true}}));
+  }
+};

aio/tools/prerender/serve.js

@@ -0,0 +1,63 @@
+'use strict';
+
+// Imports
+const fs = require('fs');
+const http = require('http');
+const path = require('path');
+const { BASE_URL, DIST_DIR, PORT } = require('./constants');
+
+// Constants
+const CONTENT_TYPES = {
+  '.css': 'text/css',
+  '.html': 'text/html',
+  '.ico': 'image/x-icon',
+  '.jpg': 'image/jpeg',
+  '.js': 'text/javascript',
+  '.json': 'application/json',
+  '.png': 'image/png',
+  '.svg': 'image/svg+xml'
+};
+const CACHE = {};
+const VERBOSE = process.argv.includes('--verbose');
+
+// Helpers
+const urlToFile = url => path.join(DIST_DIR, url);
+
+const getFile = filePath => new Promise((resolve, reject) => CACHE.hasOwnProperty(filePath) ?
+  resolve(CACHE[filePath]) :
+  fs.readFile(filePath, 'utf-8', (err, content) => err ? reject(err) : resolve(CACHE[filePath] = content)));
+
+const middleware = (req, res) => {
+  const method = req.method;
+  let url = req.url;
+
+  if (VERBOSE) console.log(`Request: ${method} ${url}`);
+  if (method !== 'GET') return;
+
+  if (url.endsWith('/')) url += 'index';
+  if (!url.includes('.')) url += '.html';
+
+  let filePath = urlToFile(url);
+  if (!fs.existsSync(filePath)) filePath = urlToFile('index.html');
+
+  getFile(filePath).
+    then(content => {
+      const contentType = CONTENT_TYPES[path.extname(filePath)] || 'application/octet-stream';
+      res.setHeader('Content-Type', contentType);
+      res.end(content);
+    }).
+    catch(err => {
+      console.error(err);
+      res.statusCode = 500;
+      res.end(http.STATUS_CODES[500]);
+    });
+};
+
+// Run
+const server = http.
+  createServer(middleware).
+  on('error', err => console.error(err)).
+  on('listening', () => console.log(`Server listening at ${BASE_URL}.`)).
+  listen(PORT);
+
+

aio/yarn.lock

@@ -254,10 +254,18 @@ ansi-html@0.0.7:
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e"
 
+ansi-regex@^0.2.0, ansi-regex@^0.2.1:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-0.2.1.tgz#0d8e946967a3d8143f93e24e298525fc1b2235f9"
+
 ansi-regex@^2.0.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df"
 
+ansi-styles@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-1.1.0.tgz#eaecbf66cd706882760b2f4691582b8f55d7a7de"
+
 ansi-styles@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
@@ -863,6 +871,16 @@ center-align@^0.1.1:
     align-text "^0.1.3"
     lazy-cache "^1.0.3"
 
+chalk@0.5.1:
+  version "0.5.1"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-0.5.1.tgz#663b3a648b68b55d04690d49167aa837858f2174"
+  dependencies:
+    ansi-styles "^1.1.0"
+    escape-string-regexp "^1.0.0"
+    has-ansi "^0.1.0"
+    strip-ansi "^0.3.0"
+    supports-color "^0.2.0"
+
 chalk@^1.0.0, chalk@^1.1.0, chalk@^1.1.1, chalk@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98"
@@ -1078,6 +1096,10 @@ combined-stream@^1.0.5, combined-stream@~1.0.5:
   dependencies:
     delayed-stream "~1.0.0"
 
+commander@2.6.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.6.0.tgz#9df7e52fb2a0cb0fb89058ee80c3104225f37e1d"
+
 commander@2.9.x, commander@^2.8.1, commander@^2.9.0:
   version "2.9.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4"
@@ -1150,6 +1172,19 @@ concat-stream@^1.4.7:
     readable-stream "^2.2.2"
     typedarray "^0.0.6"
 
+concurrently@^3.4.0:
+  version "3.4.0"
+  resolved "https://registry.yarnpkg.com/concurrently/-/concurrently-3.4.0.tgz#60662b3defde07375bae19aac0ab780ec748ba79"
+  dependencies:
+    chalk "0.5.1"
+    commander "2.6.0"
+    date-fns "^1.23.0"
+    lodash "^4.5.1"
+    rx "2.3.24"
+    spawn-command "^0.0.2-1"
+    supports-color "^3.2.3"
+    tree-kill "^1.1.0"
+
 configstore@^1.0.0, configstore@^1.2.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/configstore/-/configstore-1.4.0.tgz#c35781d0501d268c25c54b8b17f6240e8a4fb021"
@@ -1450,6 +1485,10 @@ dashdash@^1.12.0:
   dependencies:
     assert-plus "^1.0.0"
 
+date-fns@^1.23.0:
+  version "1.28.0"
+  resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.28.0.tgz#3b12f54b66467807bb95e5930caf7bfb4170bc1a"
+
 date-now@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b"
@@ -1860,7 +1899,7 @@ escape-html@~1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
 
-escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5:
+escape-string-regexp@^1.0.0, escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4"
 
@@ -2650,6 +2689,12 @@ har-validator@~2.0.6:
     is-my-json-valid "^2.12.4"
     pinkie-promise "^2.0.0"
 
+has-ansi@^0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-0.1.0.tgz#84f265aae8c0e6a88a12d7022894b7568894c62e"
+  dependencies:
+    ansi-regex "^0.2.0"
+
 has-ansi@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91"
@@ -3803,7 +3848,7 @@ lodash@^3.10.0, lodash@^3.10.1, lodash@^3.8.0, lodash@~3.10.0, lodash@~3.10.1:
   version "3.10.1"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6"
 
-lodash@^4.0.0, lodash@^4.11.1, lodash@^4.11.2, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.3.0, lodash@^4.5.0, lodash@^4.6.1:
+lodash@^4.0.0, lodash@^4.11.1, lodash@^4.11.2, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.3.0, lodash@^4.5.0, lodash@^4.5.1, lodash@^4.6.1:
   version "4.17.4"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
 
@@ -5373,6 +5418,10 @@ rx-lite@^3.1.2:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/rx-lite/-/rx-lite-3.1.2.tgz#19ce502ca572665f3b647b10939f97fd1615f102"
 
+rx@2.3.24:
+  version "2.3.24"
+  resolved "https://registry.yarnpkg.com/rx/-/rx-2.3.24.tgz#14f950a4217d7e35daa71bbcbe58eff68ea4b2b7"
+
 rx@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/rx/-/rx-4.1.0.tgz#a5f13ff79ef3b740fe30aa803fb09f98805d4782"
@@ -5554,6 +5603,14 @@ shelljs@^0.7.0:
     interpret "^1.0.0"
     rechoir "^0.6.2"
 
+shelljs@^0.7.7:
+  version "0.7.7"
+  resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.7.7.tgz#b2f5c77ef97148f4b4f6e22682e10bba8667cff1"
+  dependencies:
+    glob "^7.0.0"
+    interpret "^1.0.0"
+    rechoir "^0.6.2"
+
 sigmund@~1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/sigmund/-/sigmund-1.0.1.tgz#3ff21f198cad2175f9f3b781853fd94d0d19b590"
@@ -5690,6 +5747,10 @@ sparkles@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/sparkles/-/sparkles-1.0.0.tgz#1acbbfb592436d10bbe8f785b7cc6f82815012c3"
 
+spawn-command@^0.0.2-1:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2.tgz#9544e1a43ca045f8531aac1a48cb29bdae62338e"
+
 spdx-correct@~1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-1.0.2.tgz#4b3073d933ff51f3912f03ac5519498a4150db40"
@@ -5823,6 +5884,12 @@ stringstream@~0.0.4:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878"
 
+strip-ansi@^0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-0.3.0.tgz#25f48ea22ca79187f3174a4db8759347bb126220"
+  dependencies:
+    ansi-regex "^0.2.1"
+
 strip-ansi@^3.0.0, strip-ansi@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
@@ -5915,6 +5982,10 @@ superstatic@^4.0:
     try-require "^1.0.0"
     update-notifier "^1.0.1"
 
+supports-color@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-0.2.0.tgz#d92de2694eb3f67323973d7ae3d8b55b4c22190a"
+
 supports-color@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7"
@@ -6094,6 +6165,10 @@ toxic@^1.0.0:
   dependencies:
     lodash "^2.4.1"
 
+tree-kill@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.1.0.tgz#c963dcf03722892ec59cba569e940b71954d1729"
+
 trim-newlines@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613"