From e42bd012f91f74921caaefcfb7f859bdf7bea1d3 Mon Sep 17 00:00:00 2001 From: George Kalpakas Date: Sun, 26 Aug 2018 00:40:58 +0300 Subject: [PATCH] ci(docs-infra): test PR previews on CI (#25671) The deployment of PR previews is triggered by the notification webhook of the `aio_preview` CircleCI job (which creates and stores the build artifacts). This commit adds a new job (`test_aio_preview`), which waits for the preview to be deployed (for PRs that do have a preview) and then runs some tests against it (currently only PWA tests). Fixes #23818 PR Close #25671 --- .circleci/config.yml | 18 +++++ aio/scripts/deploy-to-firebase.sh | 2 +- aio/scripts/test-preview.js | 121 ++++++++++++++++++++++++++++++ scripts/ci/env.sh | 3 +- 4 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 aio/scripts/test-preview.js diff --git a/.circleci/config.yml b/.circleci/config.yml index b7417ad484..03a75fefe8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -158,6 +158,21 @@ jobs: # `AIO_ARTIFACT_PATH` in `aio/aio-builds-setup/Dockerfile` destination: aio/dist/aio-snapshot.tgz + # This job should only be run on PR builds, where `CIRCLE_PR_NUMBER` is defined. + test_aio_preview: + <<: *job_defaults + steps: + - checkout: + <<: *post_checkout + - restore_cache: + key: *cache_key + - run: yarn install --cwd aio --frozen-lockfile --non-interactive + - run: + name: Wait for preview and run tests + command: | + source "./scripts/ci/env.sh" print + xvfb-run --auto-servernum node scripts/test-preview.js $CIRCLE_PR_NUMBER $CIRCLE_SHA1 $AIO_MIN_PWA_SCORE + # This job exists only for backwards-compatibility with old scripts and tests # that rely on the pre-Bazel dist/packages-dist layout. # It duplicates some work with the job above: we build the bazel packages @@ -257,6 +272,9 @@ workflows: filters: branches: only: /pull\/\d+/ + - test_aio_preview: + requires: + - aio_preview - integration_test: requires: - build-packages-dist diff --git a/aio/scripts/deploy-to-firebase.sh b/aio/scripts/deploy-to-firebase.sh index 407dc5732a..2efa8ee889 100755 --- a/aio/scripts/deploy-to-firebase.sh +++ b/aio/scripts/deploy-to-firebase.sh @@ -104,5 +104,5 @@ fi firebase deploy --message "Commit: $TRAVIS_COMMIT" --non-interactive --token "$firebaseToken" # Run PWA-score tests - yarn test-pwa-score "$deployedUrl" "$MIN_PWA_SCORE" + yarn test-pwa-score "$deployedUrl" "$AIO_MIN_PWA_SCORE" ) diff --git a/aio/scripts/test-preview.js b/aio/scripts/test-preview.js new file mode 100644 index 0000000000..3130187021 --- /dev/null +++ b/aio/scripts/test-preview.js @@ -0,0 +1,121 @@ +#!/usr/bin/env node + +/** + * Usage: + * node scripts/test-preview + * + * Checks whether a PR will (eventually) have a (public) preview, waits for the preview to be + * available, and runs PWA tests against the preview. + * + * For PRs that are expected to have a preview, this script will fail if the preview is still not + * available after a pre-defined waiting period or if the PWA tests fail. + */ + +// Imports +const {spawn} = require('child_process'); +const {get: httpsGet} = require('https'); +const {relative} = require('path'); + +// Input +const [prNumber, prLastSha, minPwaScore] = validateArgs(process.argv.slice(2)); + +// Variables +const aioBuildsDomain = 'ngbuilds.io'; +const previewCheckInterval = 30000; +const previewCheckAttempts = 10; + +const shortSha = prLastSha && prLastSha.slice(0, 7); +const previewabilityCheckUrl = `https://${aioBuildsDomain}/can-have-public-preview/${prNumber}`; +const previewUrl = `https://pr${prNumber}-${shortSha}.${aioBuildsDomain}/`; + +// Check whether the PR can have a (public) preview. +get(previewabilityCheckUrl). + then(response => JSON.parse(response)). + then(({canHavePublicPreview, reason}) => { + // Nothing to do, if this PR can have no (public) preview. + if (canHavePublicPreview === false) { + reportNoPreview(reason); + return; + } + + // There should be a preview. Wait for it to be available. + return poll(previewCheckInterval, previewCheckAttempts, () => get(previewUrl)). + // The preview is still not available after the specified waiting period. + catch(() => { + const totalSecs = Math.round((previewCheckInterval * previewCheckAttempts) / 1000); + throw new Error(`Preview still not available after ${totalSecs}s.`); + }). + // The preview is now available. Run the PWA tests. + then(() => runPwaTests()); + }). + catch(onError); + +// Helpers +function get(url) { + console.log(`GET ${url}`); + return new Promise((resolve, reject) => { + const onResponse = res => { + const statusCode = res.statusCode || -1; + const isSuccess = (200 <= statusCode) && (statusCode < 400); + let responseText = ''; + + res. + on('error', reject). + on('data', d => responseText += d). + on('end', () => isSuccess ? + resolve(responseText) : + reject(`Request to '${url}' failed (status: ${statusCode}): ${responseText}`)); + }; + + httpsGet(url, onResponse). + on('error', reject); + }); +} + +function onError(err) { + console.error(err); + process.exit(1); +} + +function poll(interval, attempts, checkCondition) { + return new Promise((resolve, reject) => { + if (!attempts) return reject(); + + checkCondition(). + then(() => resolve()). + catch(() => wait(interval). + then(() => poll(interval, attempts - 1, checkCondition)). + then(resolve, reject)); + }); +} + +function reportNoPreview(reason) { + console.log(`No (public) preview available. (Reason: ${reason})`); +} + +function runPwaTests() { + return new Promise((resolve, reject) => { + const spawnOptions = {cwd: __dirname, stdio: 'inherit'}; + spawn('yarn', ['test-pwa-score', previewUrl, minPwaScore], spawnOptions). + on('error', reject). + on('exit', code => (code === 0 ? resolve : reject)()); + }); +} + +function validateArgs(args) { + if (args.length !== 3) { + const relativeScriptPath = relative('.', __filename.replace(/\.js$/, '')); + const usageCmd = `node ${relativeScriptPath} `; + + return onError( + `Invalid number of arguments (expected 3, found ${args.length}).\n` + + `Usage: ${usageCmd}`); + } + + return args; +} + +function wait(delay) { + console.log(`Waiting ${delay}ms...`); + return new Promise(resolve => setTimeout(resolve, delay)); +} diff --git a/scripts/ci/env.sh b/scripts/ci/env.sh index f65a0b0194..b90fc4e47b 100755 --- a/scripts/ci/env.sh +++ b/scripts/ci/env.sh @@ -40,6 +40,7 @@ setEnvVar CHROMIUM_VERSION 561733 # Chrome 68 linux stable, see https://www.chr setEnvVar CHROMEDRIVER_VERSION_ARG "--versions.chrome 2.41" setEnvVar SAUCE_CONNECT_VERSION 4.4.9 setEnvVar ANGULAR_CLI_VERSION 1.6.3 +setEnvVar AIO_MIN_PWA_SCORE 95 setEnvVar PROJECT_ROOT $(cd ${thisDir}/../..; pwd) if [[ ${TRAVIS:-} ]]; then @@ -63,8 +64,6 @@ if [[ ${TRAVIS:-} ]]; then # Determine the current stable branch. readonly versionRe="^\s*([0-9]+\.[0-9]+)\.[0-9]+.*$" setEnvVar STABLE_BRANCH `npm info @angular/core dist-tags.latest | sed -r "s/$versionRe/\1.x/"` - - setEnvVar MIN_PWA_SCORE 95 ;; esac else