ci(aio): add monitoring for angular.io (#22483)

This commit configures a periodic job to be run on CircleCI, performing several
checks against the actual apps deployed to production (https://angular.io) and
staging (https://next.angular.io).

Fixes #21942

PR Close #22483

.circleci/config.yml

@@ -98,9 +98,28 @@ jobs:
             - "node_modules"
             - "~/bazel_repository_cache"
 
+  aio_monitoring:
+    <<: *job_defaults
+    steps:
+      - checkout:
+          <<: *post_checkout
+      - restore_cache:
+          key: *cache_key
+      - run: xvfb-run --auto-servernum ./aio/scripts/test-production.sh
+
 workflows:
   version: 2
   default_workflow:
     jobs:
       - lint
       - build
+  aio_monitoring:
+    jobs:
+      - aio_monitoring
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master

aio/.gitignore

@@ -30,6 +30,7 @@
 /connect.lock
 /coverage
 /libpeerconnection.log
+debug.log
 npm-debug.log
 testem.log
 /typings
@@ -45,4 +46,4 @@ protractor-results*.txt
 Thumbs.db
 
 # copied dependencies
-src/assets/js/lunr*
+src/assets/js/lunr*

aio/package.json

@@ -6,6 +6,8 @@
   "author": "Angular",
   "license": "MIT",
   "scripts": {
+    "preinstall": "node ../tools/yarn/check-yarn.js",
+    "postinstall": "node tools/cli-patches/patch.js && uglifyjs node_modules/lunr/lunr.js -c -m -o src/assets/js/lunr.min.js --source-map",
     "aio-use-local": "node tools/ng-packages-installer overwrite . --debug --ignore-packages @angular/service-worker",
     "aio-use-npm": "node tools/ng-packages-installer restore .",
     "aio-check-local": "node tools/ng-packages-installer check .",
@@ -17,10 +19,9 @@
     "build-local": "yarn ~~build",
     "lint": "yarn check-env && yarn docs-lint && ng lint && yarn example-lint && yarn tools-lint",
     "test": "yarn check-env && ng test",
-    "pree2e": "yarn check-env && yarn ~~update-webdriver",
+    "pree2e": "yarn check-env && yarn update-webdriver",
     "e2e": "ng e2e --no-webdriver-update",
     "e2e-prod": "yarn e2e --environment=dev --target=production",
-    "preinstall": "node ../tools/yarn/check-yarn.js",
     "presetup": "yarn install --frozen-lockfile && yarn ~~check-env && yarn boilerplate:remove",
     "setup": "yarn aio-use-npm && yarn example-use-npm",
     "postsetup": "yarn boilerplate:add && yarn build-ie-polyfills && yarn docs",
@@ -57,12 +58,11 @@
     "generate-zips": "node ./tools/example-zipper/generateZips",
     "sw-manifest": "ngu-sw-manifest --dist dist --in ngsw-manifest.json --out dist/ngsw-manifest.json",
     "sw-copy": "cp node_modules/@angular/service-worker/bundles/worker-basic.min.js dist/",
-    "postinstall": "node tools/cli-patches/patch.js && uglifyjs node_modules/lunr/lunr.js -c -m -o src/assets/js/lunr.min.js --source-map",
     "build-ie-polyfills": "node node_modules/webpack/bin/webpack.js -p src/ie-polyfills.js src/generated/ie-polyfills.min.js",
+    "update-webdriver": "webdriver-manager update --standalone false --gecko false $CHROMEDRIVER_VERSION_ARG",
     "~~check-env": "node scripts/check-environment",
     "~~build": "ng build --target=production --environment=stable -sm",
-    "post~~build": "yarn sw-manifest && yarn sw-copy",
-    "~~update-webdriver": "webdriver-manager update --standalone false --gecko false $CHROMEDRIVER_VERSION_ARG"
+    "post~~build": "yarn sw-manifest && yarn sw-copy"
   },
   "engines": {
     "node": ">=8.9.1 <9.0.0",

aio/scripts/test-production.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set +x -eu -o pipefail
+
+(
+  readonly thisDir="$(cd $(dirname ${BASH_SOURCE[0]}); pwd)"
+  readonly aioDir="$(realpath $thisDir/..)"
+
+  readonly appPtorConf="$aioDir/tests/e2e/protractor.conf.js"
+  readonly cfgPtorConf="$aioDir/tests/deployment-config/e2e/protractor.conf.js"
+  readonly minPwaScore="95"
+  readonly urls=(
+    "https://angular.io/"
+    "https://next.angular.io"
+  )
+
+  cd "$aioDir"
+
+  # Install dependencies.
+  echo -e "\nInstalling dependencies in '$aioDir'...\n-----"
+  yarn install --frozen-lockfile
+  yarn update-webdriver
+
+  # Run checks for all URLs.
+  for url in "${urls[@]}"; do
+    echo -e "\nChecking '$url'...\n-----"
+
+    # Run e2e tests.
+    yarn protractor "$appPtorConf" --baseUrl "$url"
+
+    # Run deployment config tests.
+    yarn protractor "$cfgPtorConf" --baseUrl "$url"
+
+    # Run PWA-score tests.
+    yarn test-pwa-score "$url" "$minPwaScore"
+  done
+
+  echo -e "\nAll checks passed!"
+)

aio/tests/deployment-config/e2e/protractor.conf.js

@@ -0,0 +1,52 @@
+// Protractor configuration file, see link for more information
+// https://github.com/angular/protractor/blob/master/lib/config.ts
+
+exports.config = {
+  allScriptsTimeout: 11000,
+  specs: [
+    './*.e2e-spec.ts'
+  ],
+  capabilities: {
+    browserName: 'chrome',
+    // For Travis
+    chromeOptions: {
+      binary: process.env.CHROME_BIN,
+      args: ['--no-sandbox']
+    }
+  },
+  directConnect: true,
+  framework: 'jasmine',
+  jasmineNodeOpts: {
+    showColors: true,
+    defaultTimeoutInterval: 30000,
+    print: function() {}
+  },
+  params: {
+    sitemapUrls: [],
+    legacyUrls: [],
+  },
+  beforeLaunch() {
+    const {register} = require('ts-node');
+    register({});
+  },
+  onPrepare() {
+    const {SpecReporter} = require('jasmine-spec-reporter');
+    const {browser} = require('protractor');
+    const {loadLegacyUrls, loadRemoteSitemapUrls} = require('../shared/helpers');
+
+    return Promise.all([
+      browser.getProcessedConfig(),
+      loadRemoteSitemapUrls(browser.baseUrl),
+      loadLegacyUrls(),
+    ]).then(([config, sitemapUrls, legacyUrls]) => {
+      if (sitemapUrls.length <= 100) {
+        throw new Error(`Too few sitemap URLs. (Expected: >100 | Found: ${sitemapUrls.length})`);
+      } else if (legacyUrls.length <= 100) {
+        throw new Error(`Too few legacy URLs. (Expected: >100 | Found: ${legacyUrls.length})`);
+      }
+
+      Object.assign(config.params, {sitemapUrls, legacyUrls});
+      jasmine.getEnv().addReporter(new SpecReporter({spec: {displayStacktrace: true}}));
+    });
+  }
+};

aio/tests/deployment-config/e2e/testDeployedRedirection.e2e-spec.ts

@@ -0,0 +1,50 @@
+import { browser } from 'protractor';
+
+describe(browser.baseUrl, () => {
+  const sitemapUrls = browser.params.sitemapUrls;
+  const legacyUrls = browser.params.legacyUrls;
+  const goTo = async url => {
+    // Go to the specified URL and then unregister the ServiceWorker
+    // to ensure subsequent requests are passed through to the server.
+    await browser.get(url);
+    await browser.executeAsyncScript(cb => navigator.serviceWorker
+        .getRegistrations()
+        .then(regs => Promise.all(regs.map(reg => reg.unregister())))
+        .then(cb));
+  };
+
+  beforeAll(async done => {
+    // Make an initial request to unregister the ServiceWorker.
+    await goTo(browser.baseUrl);
+    done();
+  });
+
+  beforeEach(() => browser.waitForAngularEnabled(false));
+  afterEach(() => browser.waitForAngularEnabled(true));
+
+  describe('(with sitemap URLs)', () => {
+    sitemapUrls.forEach((url, i) => {
+      it(`should not redirect '${url}' (${i + 1}/${sitemapUrls.length})`, async () => {
+        await goTo(url);
+
+        const expectedUrl = browser.baseUrl + url;
+        const actualUrl = (await browser.getCurrentUrl()).replace(/\?.*$/, '');
+
+        expect(actualUrl).toBe(expectedUrl);
+      });
+    });
+  });
+
+  describe('(with legacy URLs)', () => {
+    legacyUrls.forEach(([fromUrl, toUrl], i) => {
+      it(`should redirect '${fromUrl}' to '${toUrl}' (${i + 1}/${legacyUrls.length})`, async () => {
+        await goTo(fromUrl);
+
+        const expectedUrl = (/^http/.test(toUrl) ? '' : browser.baseUrl.replace(/\/$/, '')) + toUrl;
+        const actualUrl = (await browser.getCurrentUrl()).replace(/\?.*$/, '');
+
+        expect(actualUrl).toBe(expectedUrl);
+      });
+    });
+  });
+});

aio/tests/deployment-config/shared/helpers.ts

@@ -1,6 +1,8 @@
 import { resolve } from 'canonical-path';
 import { load as loadJson } from 'cjson';
 import { readFileSync } from 'fs';
+import { get as httpGet } from 'http';
+import { get as httpsGet } from 'https';
 
 import { FirebaseRedirector, FirebaseRedirectConfig } from '../../../tools/firebase-test-utils/FirebaseRedirector';
 
@@ -17,20 +19,35 @@ export function loadRedirects(): FirebaseRedirectConfig[] {
   return contents.hosting.redirects;
 }
 
-export function loadSitemapUrls() {
-  const pathToSiteMap = `${AIO_DIR}/src/generated/sitemap.xml`;
-  const xml = readFileSync(pathToSiteMap, 'utf8');
-  const urls: string[] = [];
-  xml.replace(/<loc>([^<]+)<\/loc>/g, (_, loc) => urls.push(loc.replace('%%DEPLOYMENT_HOST%%', '')));
-  return urls;
-}
-
 export function loadLegacyUrls() {
   const pathToLegacyUrls = `${__dirname}/URLS_TO_REDIRECT.txt`;
   const urls = readFileSync(pathToLegacyUrls, 'utf8').split('\n').map(line => line.split('\t'));
   return urls;
 }
 
+export function loadLocalSitemapUrls() {
+  const pathToSiteMap = `${AIO_DIR}/src/generated/sitemap.xml`;
+  const xml = readFileSync(pathToSiteMap, 'utf8');
+  return extractSitemapUrls(xml);
+}
+
+export async function loadRemoteSitemapUrls(host: string) {
+  const urlToSiteMap = `${host}/generated/sitemap.xml`;
+  const get = /^https:/.test(host) ? httpsGet : httpGet;
+
+  const xml = await new Promise<string>((resolve, reject) => {
+    let responseText = '';
+    get(urlToSiteMap, res => res
+        .on('data', chunk => responseText += chunk)
+        .on('end', () => resolve(responseText))
+        .on('error', reject));
+  });
+
+  // Currently, all sitemaps use `angular.io` as host in URLs (which is fine since we only use the
+  // sitemap `angular.io`). See also `aio/src/extra-files/*/robots.txt`.
+  return extractSitemapUrls(xml, 'https://angular.io/');
+}
+
 export function loadSWRoutes() {
   const pathToSWManifest = `${AIO_DIR}/ngsw-manifest.json`;
   const contents = loadJson(pathToSWManifest);
@@ -50,3 +67,10 @@ export function loadSWRoutes() {
     }
   });
 }
+
+// Private functions
+function extractSitemapUrls(xml: string, host = '%%DEPLOYMENT_HOST%%') {
+  const urls: string[] = [];
+  xml.replace(/<loc>([^<]+)<\/loc>/g, (_, loc) => urls.push(loc.replace(host, '')) as any);
+  return urls;
+}

aio/tests/deployment-config/unit/testFirebaseRedirection.spec.ts

@@ -1,8 +1,8 @@
-import { getRedirector, loadLegacyUrls, loadRedirects, loadSitemapUrls } from '../shared/helpers';
+import { getRedirector, loadLegacyUrls, loadLocalSitemapUrls, loadRedirects } from '../shared/helpers';
 
 describe('firebase.json redirect config', () => {
   describe('with sitemap urls', () => {
-    loadSitemapUrls().forEach(url => {
+    loadLocalSitemapUrls().forEach(url => {
       it('should not redirect any urls in the sitemap', () => {
         expect(getRedirector().redirect(url)).toEqual(url);
       });

aio/tests/deployment-config/unit/testServiceWorkerRoutes.spec.ts

@@ -1,8 +1,8 @@
-import { loadLegacyUrls, loadSitemapUrls, loadSWRoutes } from '../shared/helpers';
+import { loadLegacyUrls, loadLocalSitemapUrls, loadSWRoutes } from '../shared/helpers';
 
 describe('service-worker routes', () => {
 
-  loadSitemapUrls().forEach(url => {
+  loadLocalSitemapUrls().forEach(url => {
     it('should process URLs in the Sitemap', () => {
       const routes = loadSWRoutes();
       expect(routes.some(test => test(url))).toBeTruthy(url);