From 3f68377c3dc1b61d2b2e44b1b09f4148e1b7b542 Mon Sep 17 00:00:00 2001 From: George Kalpakas Date: Mon, 25 Nov 2019 17:48:18 +0200 Subject: [PATCH] ci: check versions of non-local integration project dependencies (#33968) In order to keep integration tests on CI as determinitstic as possible, we need to ensure that the same dependencies (including transitive ones) are installed each time. One way to ensure that is using a lockfile (such as `yarn.lock`) to pin the dependencies to exact versions. This works as long as the lockfile itself is in-sync with the corresponding `package.json`, which specifies the dependencies. Ideally, we would run `yarn install` with the `--frozen-lockfile` option to verify that the lockfile is in-sync with `package.json`, but we cannot do that for integration projects, because we want to be able to install the locally built Angular packages). Therefore, we must manually esnure that the integration project lockfiles remain in-sync, which is error-prone. This commit introduces a helper script that performs some checks on each project's (non-local) dependencies: - Ensure that exact versions (not version ranges) are specified in `package.json`. This reduces the probability of installing a breaking version of a direct or transitive dependency, in case of an out-of-sync lockfile. - Ensure that the lockfile is in-sync with `package.json` wrt these dependencies. While these checks are not full-proof, they provide yet another line of defense against indeterminism. PR Close #33968 --- integration/check-dependencies.js | 79 +++++++++++++++++++++++++++++++ integration/run_tests.sh | 5 ++ package.json | 1 + yarn.lock | 2 +- 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 integration/check-dependencies.js diff --git a/integration/check-dependencies.js b/integration/check-dependencies.js new file mode 100755 index 0000000000..ce2eed8d99 --- /dev/null +++ b/integration/check-dependencies.js @@ -0,0 +1,79 @@ +#!/usr/bin/env node + +/** + * **Usage:** + * ``` + * node check-depenencies + * ``` + * + * Checks the non-local dependencies of the specified project and ensures that: + * - Exact versions (not version ranges) are specified in the project's `package.json`. + * This reduces the probability of installing a breaking version of a direct or transitive + * dependency, in case of an out-of-sync lockfile. + * - The project's lockfile (`yarn.lock`) is in-sync with `package.json` wrt these dependencies. + * + * If any of the above checks fails, the script will throw an error, otherwise it will complete + * successfully. + * + * **Context:** + * In order to keep integration tests on CI as determinitstic as possible, we need to ensure that + * the same dependencies (including transitive ones) are installed each time. One way to ensure that + * is using a lockfile (such as `yarn.lock`) to pin the dependencies to exact versions. This works + * as long as the lockfile itself is in-sync with the corresponding `package.json`, which specifies + * the dependencies. + * + * Ideally, we would run `yarn install` with the `--frozen-lockfile` option to verify that the + * lockfile is in-sync with `package.json`, but we cannot do that for integration projects, because + * we want to be able to install the locally built Angular packages). Therefore, we must manually + * esnure that the integration project lockfiles remain in-sync, which is error-prone. + * + * The checks performed by this script (although not full-proof) provide another line of defense + * against indeterminism caused by unpinned dependencies. + */ +'use strict'; + +const {parse: parseLockfile} = require('@yarnpkg/lockfile'); +const {readFileSync} = require('fs'); +const {resolve: resolvePath} = require('path'); + +const projectDir = resolvePath(process.argv[2]); +const pkgJsonPath = `${projectDir}/package.json`; +const lockfilePath = `${projectDir}/yarn.lock`; + +console.log(`Checking dependencies for '${projectDir}'...`); + +// Collect non-local dependencies (in `[name, version]` pairs). +// (Also ingore `git+https:` dependencies, because checking them is not straight-forward.) +const pkgJson = JSON.parse(readFileSync(pkgJsonPath, 'utf8')); +const deps = [ + ...Object.entries(pkgJson.dependencies || {}), + ...Object.entries(pkgJson.devDependencies || {}), +].filter(([, version]) => !/^(?:file|git\+https):/.test(version)); + +// Check for dependencies with non-exact versions. +const nonExactDeps = deps.filter(([, version]) => !/^\d+\.\d+\.\d+(?:-\w+\.\d+)?$/.test(version)); + +if (nonExactDeps.length) { + throw new Error( + `The following dependencies in '${projectDir}' are not pinned to exact versions (of the ` + + 'format X.Y.Z[-]):' + + nonExactDeps.map(([name, version]) => `\n ${name}: ${version}`)); +} + +// Check for dependencies that are not in-sync between `package.json` and the lockfile. +const {object: parsedLockfile} = parseLockfile(readFileSync(lockfilePath, 'utf8')); +const outOfSyncDeps = deps + .map(([depName, pkgJsonVersion]) => [ + depName, + pkgJsonVersion, + (parsedLockfile[`${depName}@${pkgJsonVersion}`] || {}).version, + ]) + .filter(([, pkgJsonVersion, lockfileVersion]) => pkgJsonVersion !== lockfileVersion); + +if (outOfSyncDeps.length) { + throw new Error( + `The following dependencies in '${projectDir}' are out-of-sync between 'package.json' and ` + + 'the lockfile:' + + outOfSyncDeps.map(([name, pkgJsonVersion, lockfileVersion]) => + `\n ${name}: ${pkgJsonVersion} vs ${lockfileVersion}`)); +} diff --git a/integration/run_tests.sh b/integration/run_tests.sh index d21030e53d..dac52b3bb4 100755 --- a/integration/run_tests.sh +++ b/integration/run_tests.sh @@ -55,6 +55,11 @@ for testDir in ${TEST_DIRS}; do cd $testDir rm -rf dist + # Ensure the versions of (non-local) dependencies are exact versions (not version ranges) and + # in-sync between `package.json` and the lockfile. + # (NOTE: This must be run before `yarn install`, which updates the lockfile.) + node ../check-dependencies . + yarn install --cache-folder ../$cache yarn test || exit 1 diff --git a/package.json b/package.json index c67ca55e62..0f684e884d 100644 --- a/package.json +++ b/package.json @@ -133,6 +133,7 @@ "@bazel/buildifier": "^0.29.0", "@bazel/ibazel": "^0.10.3", "@types/minimist": "^1.2.0", + "@yarnpkg/lockfile": "^1.1.0", "browserstacktunnel-wrapper": "2.0.1", "check-side-effects": "0.0.21", "clang-format": "1.0.41", diff --git a/yarn.lock b/yarn.lock index f42193a8d8..a04b4c8e1c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -874,7 +874,7 @@ resolved "https://registry.yarnpkg.com/@webcomponents/custom-elements/-/custom-elements-1.1.2.tgz#041e4c20df35245f4d160b50d044b8cff192962c" integrity sha512-gVhuQHLTrQ28v1qMp0WGPSCBukFL7qAlemxCf19TnuNZ0bO9KPF72bfhH6Hpuwdu9TptIMGNlqrr9PzqrzfZFQ== -"@yarnpkg/lockfile@1.1.0": +"@yarnpkg/lockfile@1.1.0", "@yarnpkg/lockfile@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz#e77a97fbd345b76d83245edcd17d393b1b41fb31" integrity sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==