build(aio): implement prerendering

The current implementation is based on @igorminar's [angular-io-v42][1]. It is
using Protractor to request all docs URLs, let them fallback to `/index.html`
and save the rendered page.

[1]: https://github.com/IgorMinar/angular-io-v42/tree/05508ab3/tools/prerenderer

Fixes #15104
This commit is contained in:
Georgios Kalpakas 2017-03-13 12:39:22 +02:00 committed by Miško Hevery
parent b5b2fed54d
commit d0bc83ca27
12 changed files with 320 additions and 8 deletions

1
aio/.gitignore vendored
View File

@ -7,5 +7,6 @@ yarn-error.log
# Ignore generated content
/dist
/tmp
/src/content/docs
/.sass-cache

View File

@ -35,7 +35,7 @@ server {
}
location / {
try_files $uri $uri/ /index.html =404;
try_files $uri /content/docs-prerendered/$uri.html $uri/ =404;
}
}

View File

@ -4,11 +4,10 @@
},
"hosting": {
"public": "dist",
"cleanUrls": true,
"rewrites": [
{
"source": "**",
"destination": "/index.html"
"source": "/:path*",
"destination": "/content/docs-prerendered/:path*.html"
}
]
}

View File

@ -14,13 +14,20 @@
"lint": "yarn check-env && ng lint",
"pree2e": "yarn ~~update-webdriver",
"e2e": "yarn check-env && ng e2e --no-webdriver-update",
"predeploy-preview": "yarn prerender",
"deploy-preview": "scripts/deploy-preview.sh",
"predeploy-staging": "yarn prerender",
"deploy-staging": "firebase use staging --token \"$FIREBASE_TOKEN\" && yarn ~~deploy",
"check-env": "node ../tools/check-environment.js",
"docs": "dgeni ./transforms/angular.io-package",
"docs-test": "node ../dist/tools/cjs-jasmine/index-tools ../../transforms/**/*.spec.js",
"preprerender": "yarn build",
"prerender": "concurrently --kill-others --raw --success first \"yarn ~~prerender-serve\" \"yarn ~~prerender\"",
"postprerender": "node tools/prerender/copy-to-dist",
"~~update-webdriver": "webdriver-manager update --standalone false --gecko false",
"pre~~deploy": "yarn build",
"pre~~prerender": "yarn ~~update-webdriver",
"~~prerender": "node tools/prerender/create-specs && protractor tools/prerender/protractor.conf.js",
"~~prerender-serve": "node tools/prerender/serve",
"~~deploy": "firebase deploy --message \"Commit: $TRAVIS_COMMIT\" --non-interactive --token \"$FIREBASE_TOKEN\""
},
"private": true,
@ -49,6 +56,7 @@
"@types/node": "~6.0.60",
"canonical-path": "^0.0.2",
"codelyzer": "~2.0.0-beta.4",
"concurrently": "^3.4.0",
"dgeni": "^0.4.7",
"dgeni-packages": "^0.16.8",
"entities": "^1.1.1",
@ -65,6 +73,7 @@
"lodash": "^4.17.4",
"protractor": "~5.1.0",
"rho": "^0.3.0",
"shelljs": "^0.7.7",
"ts-node": "~2.0.0",
"tslint": "~4.4.2",
"typescript": "2.1.6"

View File

@ -1,5 +1,6 @@
// Protractor configuration file, see link for more information
// https://github.com/angular/protractor/blob/master/lib/config.ts
'use strict';
/*global jasmine */
const { SpecReporter } = require('jasmine-spec-reporter');

View File

@ -11,7 +11,7 @@ UPLOAD_URL=$AIO_BUILDS_HOST/create-build/$TRAVIS_PULL_REQUEST/$TRAVIS_PULL_REQUE
cd "`dirname $0`/.."
yarn run build
# Assumes the build step has already run
tar --create --gzip --directory "$INPUT_DIR" --file "$OUTPUT_FILE" .
exec 3>&1

View File

@ -0,0 +1,30 @@
'use strict';
// Imports
const path = require('path');
// Constants
const BROWSER_INSTANCES = 7;
const PORT = 4201;
const BASE_URL = `http://localhost:${PORT}`;
const ROOT_DIR = path.join(__dirname, '../..');
const DIST_DIR = path.join(ROOT_DIR, 'dist');
const CONTENT_DIR = path.join(DIST_DIR, 'content');
const INPUT_DIR = path.join(CONTENT_DIR, 'docs');
const TMP_SPECS_DIR = path.join(ROOT_DIR, 'tmp/docs-prerender-specs');
const TMP_OUTPUT_DIR = path.join(ROOT_DIR, 'tmp/docs-prerendered');
// Exports
module.exports = {
BASE_URL,
BROWSER_INSTANCES,
CONTENT_DIR,
DIST_DIR,
INPUT_DIR,
PORT,
ROOT_DIR,
TMP_OUTPUT_DIR,
TMP_SPECS_DIR
};

View File

@ -0,0 +1,10 @@
'use strict';
// Imports
const sh = require('shelljs');
const { CONTENT_DIR, TMP_OUTPUT_DIR } = require('./constants');
sh.config.fatal = true;
// Run
sh.cp('-r', TMP_OUTPUT_DIR, CONTENT_DIR);

View File

@ -0,0 +1,89 @@
'use strict';
// Imports
const fs = require('fs');
const path = require('path');
const sh = require('shelljs');
const { BASE_URL, BROWSER_INSTANCES, INPUT_DIR, PORT, TMP_OUTPUT_DIR, TMP_SPECS_DIR } = require('./constants');
sh.config.fatal = true;
// Helpers
const chunkArray = (items, numChunks) => {
numChunks = Math.min(numChunks, items.length);
const itemsPerChunk = Math.ceil(items.length / numChunks);
const chunks = new Array(numChunks);
console.log(`Chunking ${items.length} items into ${numChunks} chunks.`);
for (let i = 0; i < numChunks; i++) {
chunks[i] = items.slice(i * itemsPerChunk, (i + 1) * itemsPerChunk);
}
return chunks;
};
const getAllFiles = rootDir => fs.readdirSync(rootDir).reduce((files, file) => {
const absolutePath = path.join(rootDir, file);
const isFile = fs.lstatSync(absolutePath).isFile();
return files.concat(isFile ? absolutePath : getAllFiles(absolutePath));
}, []);
const getAllUrls = rootDir => getAllFiles(rootDir).
filter(absolutePath => path.extname(absolutePath) === '.json').
map(absolutePath => absolutePath.slice(0, -5)).
map(absolutePath => path.relative(INPUT_DIR, absolutePath)).
map(relativePath => `${BASE_URL}/${relativePath}`);
const getTestForChunk = (chunk, idx) => `
'use strict';
const fs = require('fs');
const path = require('path');
const protractor = require('protractor');
const sh = require('shelljs');
const url = require('url');
const browser = protractor.browser;
sh.config.fatal = true;
describe('chunk ${idx}', () => ${JSON.stringify(chunk)}.forEach(urlToPage => {
const parsedUrl = url.parse(urlToPage);
it(\`should render \${parsedUrl.path}\`, done => {
browser.get(urlToPage);
browser.getPageSource()
.then(source => {
if (/document not found/i.test(source) && !/file-not-found/i.test(urlToPage)) {
return Promise.reject(\`404 for \${urlToPage}\`);
}
const relativeFilePath = parsedUrl.path.replace(/\\/$/, '/index').replace(/^\\//, '') + '.html';
const absoluteFilePath = path.resolve('${TMP_OUTPUT_DIR}', relativeFilePath);
const absoluteDirPath = path.dirname(absoluteFilePath);
console.log(\`Writing to \${absoluteFilePath}...\`);
sh.mkdir('-p', absoluteDirPath);
fs.writeFileSync(absoluteFilePath, source);
})
.then(done, done.fail);
});
}));
`;
// Run
const docsUrls = getAllUrls(INPUT_DIR);
const chunked = chunkArray(docsUrls, BROWSER_INSTANCES);
sh.rm('-rf', TMP_OUTPUT_DIR);
sh.rm('-rf', TMP_SPECS_DIR);
sh.mkdir('-p', TMP_SPECS_DIR);
chunked.forEach((chunk, idx) => {
const outputFile = path.join(TMP_SPECS_DIR, `chunk${idx}.spec.js`);
const testContent = getTestForChunk(chunk, idx);
fs.writeFileSync(outputFile, testContent);
});

View File

@ -0,0 +1,35 @@
// Protractor configuration file, see link for more information
// https://github.com/angular/protractor/blob/master/lib/config.ts
'use strict';
/*global jasmine */
const { SpecReporter } = require('jasmine-spec-reporter');
const path = require('path');
const { BASE_URL, BROWSER_INSTANCES, TMP_SPECS_DIR } = require('./constants');
exports.config = {
allScriptsTimeout: 11000,
specs: [
path.join(TMP_SPECS_DIR, 'chunk*.spec.js')
],
capabilities: {
browserName: 'chrome',
shardTestFiles: true,
maxInstances: BROWSER_INSTANCES,
// For Travis
chromeOptions: {
binary: process.env.CHROME_BIN
}
},
directConnect: true,
baseUrl: BASE_URL,
framework: 'jasmine',
jasmineNodeOpts: {
showColors: true,
defaultTimeoutInterval: 30000,
print: function() {}
},
onPrepare() {
jasmine.getEnv().addReporter(new SpecReporter({spec: {displayStacktrace: true}}));
}
};

View File

@ -0,0 +1,63 @@
'use strict';
// Imports
const fs = require('fs');
const http = require('http');
const path = require('path');
const { BASE_URL, DIST_DIR, PORT } = require('./constants');
// Constants
const CONTENT_TYPES = {
'.css': 'text/css',
'.html': 'text/html',
'.ico': 'image/x-icon',
'.jpg': 'image/jpeg',
'.js': 'text/javascript',
'.json': 'application/json',
'.png': 'image/png',
'.svg': 'image/svg+xml'
};
const CACHE = {};
const VERBOSE = process.argv.includes('--verbose');
// Helpers
const urlToFile = url => path.join(DIST_DIR, url);
const getFile = filePath => new Promise((resolve, reject) => CACHE.hasOwnProperty(filePath) ?
resolve(CACHE[filePath]) :
fs.readFile(filePath, 'utf-8', (err, content) => err ? reject(err) : resolve(CACHE[filePath] = content)));
const middleware = (req, res) => {
const method = req.method;
let url = req.url;
if (VERBOSE) console.log(`Request: ${method} ${url}`);
if (method !== 'GET') return;
if (url.endsWith('/')) url += 'index';
if (!url.includes('.')) url += '.html';
let filePath = urlToFile(url);
if (!fs.existsSync(filePath)) filePath = urlToFile('index.html');
getFile(filePath).
then(content => {
const contentType = CONTENT_TYPES[path.extname(filePath)] || 'application/octet-stream';
res.setHeader('Content-Type', contentType);
res.end(content);
}).
catch(err => {
console.error(err);
res.statusCode = 500;
res.end(http.STATUS_CODES[500]);
});
};
// Run
const server = http.
createServer(middleware).
on('error', err => console.error(err)).
on('listening', () => console.log(`Server listening at ${BASE_URL}.`)).
listen(PORT);

View File

@ -254,10 +254,18 @@ ansi-html@0.0.7:
version "0.0.7"
resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e"
ansi-regex@^0.2.0, ansi-regex@^0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-0.2.1.tgz#0d8e946967a3d8143f93e24e298525fc1b2235f9"
ansi-regex@^2.0.0:
version "2.1.1"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df"
ansi-styles@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-1.1.0.tgz#eaecbf66cd706882760b2f4691582b8f55d7a7de"
ansi-styles@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
@ -863,6 +871,16 @@ center-align@^0.1.1:
align-text "^0.1.3"
lazy-cache "^1.0.3"
chalk@0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-0.5.1.tgz#663b3a648b68b55d04690d49167aa837858f2174"
dependencies:
ansi-styles "^1.1.0"
escape-string-regexp "^1.0.0"
has-ansi "^0.1.0"
strip-ansi "^0.3.0"
supports-color "^0.2.0"
chalk@^1.0.0, chalk@^1.1.0, chalk@^1.1.1, chalk@^1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98"
@ -1078,6 +1096,10 @@ combined-stream@^1.0.5, combined-stream@~1.0.5:
dependencies:
delayed-stream "~1.0.0"
commander@2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.6.0.tgz#9df7e52fb2a0cb0fb89058ee80c3104225f37e1d"
commander@2.9.x, commander@^2.8.1, commander@^2.9.0:
version "2.9.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4"
@ -1150,6 +1172,19 @@ concat-stream@^1.4.7:
readable-stream "^2.2.2"
typedarray "^0.0.6"
concurrently@^3.4.0:
version "3.4.0"
resolved "https://registry.yarnpkg.com/concurrently/-/concurrently-3.4.0.tgz#60662b3defde07375bae19aac0ab780ec748ba79"
dependencies:
chalk "0.5.1"
commander "2.6.0"
date-fns "^1.23.0"
lodash "^4.5.1"
rx "2.3.24"
spawn-command "^0.0.2-1"
supports-color "^3.2.3"
tree-kill "^1.1.0"
configstore@^1.0.0, configstore@^1.2.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/configstore/-/configstore-1.4.0.tgz#c35781d0501d268c25c54b8b17f6240e8a4fb021"
@ -1450,6 +1485,10 @@ dashdash@^1.12.0:
dependencies:
assert-plus "^1.0.0"
date-fns@^1.23.0:
version "1.28.0"
resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.28.0.tgz#3b12f54b66467807bb95e5930caf7bfb4170bc1a"
date-now@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b"
@ -1860,7 +1899,7 @@ escape-html@~1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5:
escape-string-regexp@^1.0.0, escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4"
@ -2650,6 +2689,12 @@ har-validator@~2.0.6:
is-my-json-valid "^2.12.4"
pinkie-promise "^2.0.0"
has-ansi@^0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-0.1.0.tgz#84f265aae8c0e6a88a12d7022894b7568894c62e"
dependencies:
ansi-regex "^0.2.0"
has-ansi@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91"
@ -3803,7 +3848,7 @@ lodash@^3.10.0, lodash@^3.10.1, lodash@^3.8.0, lodash@~3.10.0, lodash@~3.10.1:
version "3.10.1"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6"
lodash@^4.0.0, lodash@^4.11.1, lodash@^4.11.2, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.3.0, lodash@^4.5.0, lodash@^4.6.1:
lodash@^4.0.0, lodash@^4.11.1, lodash@^4.11.2, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.3.0, lodash@^4.5.0, lodash@^4.5.1, lodash@^4.6.1:
version "4.17.4"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
@ -5373,6 +5418,10 @@ rx-lite@^3.1.2:
version "3.1.2"
resolved "https://registry.yarnpkg.com/rx-lite/-/rx-lite-3.1.2.tgz#19ce502ca572665f3b647b10939f97fd1615f102"
rx@2.3.24:
version "2.3.24"
resolved "https://registry.yarnpkg.com/rx/-/rx-2.3.24.tgz#14f950a4217d7e35daa71bbcbe58eff68ea4b2b7"
rx@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/rx/-/rx-4.1.0.tgz#a5f13ff79ef3b740fe30aa803fb09f98805d4782"
@ -5554,6 +5603,14 @@ shelljs@^0.7.0:
interpret "^1.0.0"
rechoir "^0.6.2"
shelljs@^0.7.7:
version "0.7.7"
resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.7.7.tgz#b2f5c77ef97148f4b4f6e22682e10bba8667cff1"
dependencies:
glob "^7.0.0"
interpret "^1.0.0"
rechoir "^0.6.2"
sigmund@~1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/sigmund/-/sigmund-1.0.1.tgz#3ff21f198cad2175f9f3b781853fd94d0d19b590"
@ -5690,6 +5747,10 @@ sparkles@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/sparkles/-/sparkles-1.0.0.tgz#1acbbfb592436d10bbe8f785b7cc6f82815012c3"
spawn-command@^0.0.2-1:
version "0.0.2"
resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2.tgz#9544e1a43ca045f8531aac1a48cb29bdae62338e"
spdx-correct@~1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-1.0.2.tgz#4b3073d933ff51f3912f03ac5519498a4150db40"
@ -5823,6 +5884,12 @@ stringstream@~0.0.4:
version "0.0.5"
resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878"
strip-ansi@^0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-0.3.0.tgz#25f48ea22ca79187f3174a4db8759347bb126220"
dependencies:
ansi-regex "^0.2.1"
strip-ansi@^3.0.0, strip-ansi@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
@ -5915,6 +5982,10 @@ superstatic@^4.0:
try-require "^1.0.0"
update-notifier "^1.0.1"
supports-color@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-0.2.0.tgz#d92de2694eb3f67323973d7ae3d8b55b4c22190a"
supports-color@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7"
@ -6094,6 +6165,10 @@ toxic@^1.0.0:
dependencies:
lodash "^2.4.1"
tree-kill@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.1.0.tgz#c963dcf03722892ec59cba569e940b71954d1729"
trim-newlines@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613"