From 4fc41517d1ccc13a0a3369a9a2c9027ada982aff Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Sun, 6 Jan 2019 12:37:46 +0100 Subject: [PATCH] build: shard integration tests on circleci (#27937) PR Close #27937 --- .circleci/config.yml | 5 +- integration/get-sharded-tests.js | 100 +++++++++++++++++++++++++++++++ integration/run_tests.sh | 17 +++++- 3 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 integration/get-sharded-tests.js diff --git a/.circleci/config.yml b/.circleci/config.yml index 97b332792a..4ea66435ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -333,6 +333,7 @@ jobs: # See comments inside the integration/run_tests.sh script. integration_test: <<: *job_defaults + parallelism: 4 docker: # Needed because the integration tests expect Chrome to be installed (e.g cli-hello-world) - image: *browsers_docker_image @@ -348,7 +349,9 @@ jobs: - attach_workspace: at: dist - *define_env_vars - - run: ./integration/run_tests.sh + # Runs the integration tests in parallel across multiple CircleCI container instances. The + # amount of container nodes for this job is controlled by the "parallelism" option. + - run: ./integration/run_tests.sh ${CIRCLE_NODE_INDEX} ${CIRCLE_NODE_TOTAL} # This job updates the content of repos like github.com/angular/core-builds # for every green build on angular/angular. diff --git a/integration/get-sharded-tests.js b/integration/get-sharded-tests.js new file mode 100644 index 0000000000..4039e4dbec --- /dev/null +++ b/integration/get-sharded-tests.js @@ -0,0 +1,100 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +/* + * Script that determines the sharded tests for the current CircleCI container. CircleCI starts + * multiple containers if the "parallelism" option has been specified and this script splits up + * the integration tests into shards based on the amount of parallelism. + * + * It's also possible to manually specify tests which should run on a container because some + * integration tests are more complex and take up more time. In order to properly balance the + * duration of each container, we allow manual test shards to be specified. + * + * The output of this script can then be used to only run the tests which are assigned to the + * current CircleCI container. + */ + +const fs = require('fs'); +const path = require('path'); +const minimist = require('minimist'); + +// Parsed command line arguments. +const {shardIndex, maxShards} = minimist(process.argv.slice(2)); + +// Ensure that all CLI options are set properly. +if (shardIndex == null) { + throw new Error('The "--shardIndex" option has not been specified.') +} else if (maxShards == null) { + throw new Error('The "--maxShards" option has not been specified.'); +} + +// List of all integration tests that are available. +const integrationTests = fs.readdirSync(__dirname).filter( + testName => fs.statSync(path.join(__dirname, testName)).isDirectory()); + +// Manual test shards which aren't computed automatically. This is helpful when a specific +// set of integration test takes up *way* more time than all other tests, and we want to +// balance out the duration for all specific shards. +const manualTestShards = [ + // The first shard should only run the bazel integration tests because these take up + // a lot of time and shouldn't be split up automatically. + ['bazel', 'bazel-schematics'] +]; + +// Tests which haven't been assigned manually to a shard. These tests will be automatically +// split across the remaining available shards. +const unassignedTests = stripManualOverrides(integrationTests, manualTestShards); + +if (manualTestShards.length === maxShards && unassignedTests.length) { + throw new Error( + `Tests have been specified manually for all available shards, but there were ` + + `integration tests which haven't been specified and won't run right now. Missing ` + + `tests: ${unassignedTests.join(', ')}`) +} else if (manualTestShards.length > maxShards) { + throw new Error( + `Too many manual shards have been specified. Increase the amount of maximum shards.`); +} + +// In case the shard for the current index has been specified manually, we just output +// the tests for the manual shard. +if (manualTestShards[shardIndex]) { + printTestNames(manualTestShards[shardIndex]); +} else { + const amountManualShards = manualTestShards.length; + // In case there isn't a manual shard specified for this shard index, we just compute the + // tests for this shard. Note that we need to subtract the amount of manual shards because + // we need to split up the unassigned tests across the remaining available shards. + printTestNames(getTestsForShardIndex( + unassignedTests, shardIndex - amountManualShards, maxShards - amountManualShards)); +} + +/** + * Splits the specified tests into a limited amount of shards and returns the tests that should + * run on the given shard. The shards of tests are being created deterministically and therefore + * we get reproducible tests when executing the same script multiple times. + */ +function getTestsForShardIndex(tests, shardIndex, maxShards) { + return tests.filter((n, index) => index % maxShards === shardIndex); +} + +/** + * Strips all manual tests from the list of integration tests. This is necessary because + * when computing the shards automatically we don't want to include manual tests again. This + * would mean that CircleCI runs some integration tests multiple times. + */ +function stripManualOverrides(integrationTests, manualShards) { + const allManualTests = manualShards.reduce((res, manualTests) => res.concat(manualTests), []); + return integrationTests.filter(testName => !allManualTests.includes(testName)) +} + +/** Prints the specified test names to the stdout. */ +function printTestNames(testNames) { + // Print the test names joined with spaces because this allows Bash to easily convert the output + // of this script into an array. + process.stdout.write(testNames.join(' ')); +} diff --git a/integration/run_tests.sh b/integration/run_tests.sh index 59f8d5f489..f327287300 100755 --- a/integration/run_tests.sh +++ b/integration/run_tests.sh @@ -10,7 +10,10 @@ cd "$(dirname "$0")" # basedir is the workspace root readonly basedir=$(pwd)/.. -# Track payload size functions +# When running on the CI, we track the payload size of various integration output files. Also +# we shard tests across multiple CI job instances. The script needs to be run with a shard index +# and the maximum amount of shards available for the integration tests on the CI. +# For example: "./run_tests.sh {SHARD_INDEX} {MAX_SHARDS}". if $CI; then # We don't install this by default because it contains some broken Bazel setup # and also it's a very big dependency that we never use except when publishing @@ -18,12 +21,22 @@ if $CI; then yarn add --silent -D firebase-tools@5.1.1 source ${basedir}/scripts/ci/payload-size.sh + SHARD_INDEX=${1:?"No shard index has been specified."} + MAX_SHARDS=${2:?"The maximum amount of shards has not been specified."} + + # Determines the tests that need to be run for this shard index. + TEST_DIRS=$(node ./get-sharded-tests.js --shardIndex ${SHARD_INDEX} --maxShards ${MAX_SHARDS}) + # NB: we don't run build-packages-dist.sh because we expect that it was done # by an earlier job in the CircleCI workflow. else # Not on CircleCI so let's build the packages-dist directory. # This should be fast on incremental re-build. ${basedir}/scripts/build-packages-dist.sh + + # If we aren't running on CircleCI, we do not shard tests because this would be the job of + # Bazel eventually. For now, we just run all tests sequentially when running locally. + TEST_DIRS=$(ls | grep -v node_modules) fi # Workaround https://github.com/yarnpkg/yarn/issues/2165 @@ -36,7 +49,7 @@ rm_cache mkdir $cache trap rm_cache EXIT -for testDir in $(ls | grep -v node_modules) ; do +for testDir in ${TEST_DIRS}; do [[ -d "$testDir" ]] || continue echo "#################################" echo "Running integration test $testDir"