refactor: simplify bazel saucelabs targets using karma pre-test wrapper and shared saucelabs connection between tests (#34769)

* Added a /tools/saucelabs/sauce-service.sh script that manages the sauce-connect as a service which is used by the karma-saucelabs.js wrapper to start the service.
* Added /tools/saucelabs/README.md that covers the details of SauceLabs karma testing with Bazel.

PR Close #34769
This commit is contained in:
Greg Magolan 2020-01-13 14:40:21 -08:00 committed by Andrew Kushnir
parent a5a598104f
commit dff4e1e19c
13 changed files with 672 additions and 374 deletions

View File

@ -112,6 +112,16 @@ build:remote --project_id=internal-200822
build:remote --remote_cache=remotebuildexecution.googleapis.com
build:remote --remote_executor=remotebuildexecution.googleapis.com
##################################
# Saucelabs tests settings #
# Turn on these settings with #
# --config=saucelabs #
##################################
# For saucelabs tests we don't want to enable flaky test attempts. Karma has its own integrated
# retry mechanism and we do not want to retry unnecessarily if Karma already tried multiple times.
test:saucelabs --flaky_test_attempts=1
###############################
# NodeJS rules settings
# These settings are required for rules_nodejs

View File

@ -13,7 +13,3 @@ test --flaky_test_attempts=2
# More details on failures
build --verbose_failures=true
# For saucelabs tests we don't want to enable flaky test attempts. Karma has its own integrated
# retry mechanism and we do not want to retry unnecessarily if Karma already tried multiple times.
test:saucelabs --flaky_test_attempts=1

View File

@ -314,19 +314,15 @@ jobs:
- custom_attach_workspace
- init_environment
- run:
name: Run Bazel tests in saucelabs
# All web tests are contained within a single //:saucelabs_unit_tests_poc target
# for Saucelabs as running each set of tests as a separate target will attempt to acquire
# too many browsers on Saucelabs (7 per target currently) and some tests will always
# fail to acquire browsers. For example:
# 14 02 2019 19:52:33.170:WARN [launcher]: chrome beta on SauceLabs have not captured in 180000 ms, killing.
# //packages/forms/test:web_test_sauce TIMEOUT in 315.0s
command: |
./scripts/saucelabs/run-bazel-via-tunnel.sh \
--tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \
--username $SAUCE_USERNAME \
--key $(echo $SAUCE_ACCESS_KEY | rev) \
-- yarn bazel test //:saucelabs_unit_tests_poc --config=saucelabs
name: Preparing environment for running tests on Saucelabs.
command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
- run:
name: Run Bazel tests on Saucelabs
# See /tools/saucelabs/README.md for more info
command: |
yarn bazel run //tools/saucelabs:sauce_service_setup
yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs
yarn bazel run //tools/saucelabs:sauce_service_stop
no_output_timeout: 20m
- notify_webhook_on_fail:
webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL
@ -341,15 +337,16 @@ jobs:
steps:
- custom_attach_workspace
- init_environment
- run:
name: Preparing environment for running tests on Saucelabs.
command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
- run:
name: Run Bazel tests on Saucelabs
# Runs the //:saucelabs_tests target with Saucelabs and Ivy.
command: |
./scripts/saucelabs/run-bazel-via-tunnel.sh \
--tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \
--username $SAUCE_USERNAME \
--key $(echo $SAUCE_ACCESS_KEY | rev) \
-- yarn bazel test //:saucelabs_unit_tests --config=ivy --config=saucelabs
# See /tools/saucelabs/README.md for more info
command: |
yarn bazel run //tools/saucelabs:sauce_service_setup
yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy
yarn bazel run //tools/saucelabs:sauce_service_stop
no_output_timeout: 20m
test_aio:
@ -665,17 +662,20 @@ jobs:
setPublicVar KARMA_JS_BROWSERS $(node -e 'console.log(require("./browser-providers.conf").sauceAliases.CI_REQUIRED.join(","))')
setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
- run:
name: Starting Saucelabs tunnel
command: ./scripts/saucelabs/start-tunnel.sh
name: Starting Saucelabs tunnel service
command: ./tools/saucelabs/sauce-service.sh run
background: true
- run: yarn tsc -p packages
- run: yarn tsc -p modules
- run: yarn bazel build //packages/zone.js:npm_package
# Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
# too early without Saucelabs not being ready.
- run: ./scripts/saucelabs/wait-for-tunnel.sh
- run:
# Waiting on ready ensures that we don't run tests too early without Saucelabs not being ready.
name: Waiting for Saucelabs tunnel to connect
command: ./tools/saucelabs/sauce-service.sh ready-wait
- run: yarn karma start ./karma-js.conf.js --single-run --browsers=${KARMA_JS_BROWSERS}
- run: ./scripts/saucelabs/stop-tunnel.sh
- run:
name: Stop Saucelabs tunnel service
command: ./tools/saucelabs/sauce-service.sh stop
# Job that runs all unit tests of the `angular/components` repository.
components-repo-unit-tests:

View File

@ -47,12 +47,15 @@ filegroup(
],
)
# To run a karma_web_test target manually, run the "./scripts/saucelabs/run-bazel-via-tunnel.sh"
# script. Note: If you are on MacOS or Windows, you need to manually start the Saucelabs tunnel
# because the script only supports the linux Saucelabs tunnel binary. We combine all tests into
# a single "karma_web_test" target because running them as separate targets in parallel can result
# in to too many browsers being acquired at the same time. This will then result in tests being
# flaky. This target runs in CI with Saucelabs and Ivy.
# To run manually:
# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER.
# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary.
# environment variables and run:
# ```
# yarn bazel run //tools/saucelabs:sauce_service_setup
# yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy
# ```
# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel.
karma_web_test(
name = "saucelabs_unit_tests",
# Default timeout is moderate (5min). This causes the test to be terminated while
@ -60,9 +63,10 @@ karma_web_test(
# unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
# Bazel's long timeout (15min). This ensures that Karma can shut down properly.
timeout = "long",
karma = "//tools/saucelabs:karma-saucelabs",
tags = [
"local",
"manual",
"no-remote-exec",
"saucelabs",
],
deps = [
@ -70,45 +74,48 @@ karma_web_test(
],
)
karma_web_test(
# This target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a
SAUCE_TEST_SUITE_TARGETS = [
"packages/common/http/test:test_lib",
"packages/common/http/testing/test:test_lib",
"packages/common/test:test_lib",
"packages/core/test:test_lib",
"packages/forms/test:test_lib",
"packages/http/test:test_lib",
]
[
# These target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a
# subset of the legacy saucelabs tests.
name = "saucelabs_unit_tests_poc",
# Default timeout is moderate (5min). This causes the test to be terminated while
# Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
# unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
# Bazel's long timeout (15min). This ensures that Karma can shut down properly.
timeout = "long",
tags = [
"local",
"manual",
"saucelabs",
],
deps = [
# We combine all tests into a single karma_web_test target
# as running them as separate targets in parallel leads to too many
# browsers being acquired at once in SauceLabs and the tests flake out
# TODO: this is an example subset of tests below, add all remaining angular tests
"//packages/common/http/test:test_lib",
"//packages/common/http/testing/test:test_lib",
"//packages/common/test:test_lib",
"//packages/core/test:test_lib",
"//packages/forms/test:test_lib",
"//packages/http/test:test_lib",
"//packages/zone.js/test:karma_jasmine_test_ci",
# "//packages/router/test:test_lib",
# //packages/router/test:test_lib fails with:
# IE 11.0.0 (Windows 8.1.0.0) bootstrap should restore the scrolling position FAILED
# Expected undefined to equal 5000.
# at stack (eval code:2338:11)
# at buildExpectationResult (eval code:2305:5)
# at expectationResultFactory (eval code:858:11)
# at Spec.prototype.addExpectationResult (eval code:487:5)
# at addExpectationResult (eval code:802:9)
# at Anonymous function (eval code:2252:7)
# at Anonymous function (eval code:339:25)
# at step (eval code:133:17)
# at Anonymous function (eval code:114:50)
# at fulfilled (eval code:104:47)
],
karma_web_test(
name = "saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_"),
# Default timeout is moderate (5min). This causes the test to be terminated while
# Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
# unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
# Bazel's long timeout (15min). This ensures that Karma can shut down properly.
timeout = "long",
karma = "//tools/saucelabs:karma-saucelabs",
tags = [
"exclusive",
"manual",
"no-remote-exec",
"saucelabs",
],
deps = ["//%s" % test],
)
for test in SAUCE_TEST_SUITE_TARGETS
]
# To run manually:
# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER.
# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary.
# environment variables and run:
# ```
# yarn bazel run //tools/saucelabs:sauce_service_setup
# yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs
# ```
# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel.
test_suite(
name = "saucelabs_unit_tests_poc_suite",
tags = ["manual"],
tests = ["//:saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_") for test in SAUCE_TEST_SUITE_TARGETS],
)

View File

@ -152,9 +152,6 @@ module.exports = function(config) {
set: () => {},
});
// When running under Bazel with karma_web_test, SAUCE_TUNNEL_IDENTIFIER and KARMA_WEB_TEST_MODE
// will only be available if they are part of the Bazel action environment. More details in the
// "scripts/saucelabs/run-bazel-via-tunnel.sh" script.
if (process.env['SAUCE_TUNNEL_IDENTIFIER']) {
console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`);

View File

@ -1,204 +0,0 @@
#!/usr/bin/env bash
set -u -e -o pipefail
# Prints out usage information for the script.
function printUsage {
echo -e "\e[1mrun-bazel-via-tunnel.sh\e[0m - Runs a bazel command using a saucelabs tunnel
\e[1mUsage:\e[0m $0 --tunnel-id=<tunnel_id> \\
--username=<saucelabs_username> --key=<saucelabs_key> <bazel command>
\e[1mExample:\e[0m ./run-bazel-via-tunnel.sh --tunnel-id=<tunnel_id> \\
--username=<saucelabs_username> --key=<saucelabs_key> \\
yarn bazel test //src:everything
Flags:
--username: The saucelabs username
--key: The saucelabs access key
--tunnel-id: An identifier for the saucelabs tunnel";
}
# Ensures a file is created, creating directories for the full path as needed.
function touch-safe {
for f in "$@"; do
[ -d $f:h ] || mkdir -p $f:h && command touch $f
done
}
# The root directory of the git project the script is running in.
readonly GIT_ROOT_DIR=$(git rev-parse --show-toplevel 2> /dev/null)
# Location for the saucelabs log file.
readonly SAUCE_LOG_FILE=/tmp/angular/sauce-connect.log
# Location for the saucelabs ready to connect lock file.
readonly SAUCE_READY_FILE=/tmp/angular/sauce-connect-ready-file.lock
# Location for the saucelabs ready to connection process id lock file.
readonly SAUCE_PID_FILE=/tmp/angular/sauce-connect-pid-file.lock
# Amount of seconds we wait for sauceconnect to establish a tunnel instance. In order to not
# acquire CircleCI instances for too long if sauceconnect failed, we need a connect timeout.
readonly SAUCE_READY_FILE_TIMEOUT=120
# Create saucelabs log file if it doesn't already exist.
touch-safe $SAUCE_LOG_FILE;
# Handle configuration of script from command line flags and arguments
OPTIONS=$(getopt -u -l tunnel-id:,username:,key:,help --options "" -- "$@")
# Exit if flag parsing fails.
if [ $? != 0 ] ; then echo "Failed to parse flags, exiting" && printUsage >&2 ; exit 1 ; fi
set -- $OPTIONS
while true; do
case "$1" in
--tunnel-id)
shift
SAUCE_TUNNEL_IDENTIFIER=$1
;;
--username)
shift
SAUCE_USERNAME=$1
;;
--key)
shift
SAUCE_ACCESS_KEY=$1
;;
--help)
printUsage
exit 2
;;
--)
shift
USER_COMMAND=$@
break
;;
*)
shift
;;
esac
done
# Check each required flag and parameter
if [[ -z ${SAUCE_TUNNEL_IDENTIFIER+x} ]]; then
echo "Missing required flag: --tunnel-id"
badCommandSyntax=1
fi
if [[ -z ${SAUCE_USERNAME+x} ]]; then
echo "Missing required flag: --username"
badCommandSyntax=1
fi
if [[ -z ${SAUCE_ACCESS_KEY+x} ]]; then
echo "Missing required flag: --key"
badCommandSyntax=1
fi
if [[ "${USER_COMMAND}" == "" ]]; then
echo "Missing required bazel command: Bazel command for running in saucelabs tunnel"
badCommandSyntax=1
elif [[ ! $USER_COMMAND =~ ^(yarn bazel) ]]; then
echo "The command provided must be a bazel command run via yarn, beginning with \"yarn bazel\""
badCommandSyntax=1
fi
# If any required flag or parameter were found to be missing or incorrect, exit the script.
if [[ ${badCommandSyntax+x} ]]; then
echo
printUsage
exit 1
fi
# Command arguments that will be passed to sauce-connect.
# By default we disable SSL bumping for all requests. This is because SSL bumping is
# not needed for our test setup and in order to perform the SSL bumping, Saucelabs
# intercepts all HTTP requests in the tunnel VM and modifies them. This can cause
# flakiness as it makes all requests dependent on the SSL bumping middleware.
# See: https://wiki.saucelabs.com/display/DOCS/Troubleshooting+Sauce+Connect#TroubleshootingSauceConnect-DisablingSSLBumping
sauceArgs="--no-ssl-bump-domains all"
sauceArgs="${sauceArgs} --logfile ${SAUCE_LOG_FILE}"
sauceArgs="${sauceArgs} --readyfile ${SAUCE_READY_FILE}"
sauceArgs="${sauceArgs} --pidfile ${SAUCE_PID_FILE}"
sauceArgs="${sauceArgs} --tunnel-identifier ${SAUCE_TUNNEL_IDENTIFIER}"
sauceArgs="${sauceArgs} -u ${SAUCE_USERNAME}"
#########################
# Open saucelabs tunnel #
#########################
${GIT_ROOT_DIR}/node_modules/sauce-connect/bin/sc -k $SAUCE_ACCESS_KEY ${sauceArgs} &
########################################
# Wait for saucelabs tunnel to connect #
########################################
counter=0
while [[ ! -f ${SAUCE_READY_FILE} ]]; do
counter=$((counter + 1))
# Counter needs to be multiplied by two because the while loop only sleeps a half second.
# This has been made in favor of better progress logging (printing dots every half second)
if [ $counter -gt $[${SAUCE_READY_FILE_TIMEOUT} * 2] ]; then
echo "Timed out after ${SAUCE_READY_FILE_TIMEOUT} seconds waiting for tunnel ready file."
echo "Printing logfile output:"
echo ""
cat ${SAUCE_LOG_FILE}
exit 5
fi
printf "."
sleep 0.5
done
#########################
# Execute Bazel command #
#########################
# Prevent immediate exit for Bazel test failures
set +e
(
cd $GIT_ROOT_DIR && \
# Run bazel command with saucelabs specific environment variables passed to the action
# The KARMA_WEB_TEST_MODE and SAUCE_TUNNEL_IDENTIFIER environment variables provide
# environment variables to be read in the karma configuration file to set correct
# configurations for karma saucelabs and browser configs.
# Usage of these environment variables can be seen in this repo in
# /karma-js.conf.js and /browser-providers.conf.js
eval "$USER_COMMAND --define=KARMA_WEB_TEST_MODE=SL_REQUIRED \
--action_env=SAUCE_USERNAME=$SAUCE_USERNAME \
--action_env=SAUCE_ACCESS_KEY=$SAUCE_ACCESS_KEY \
--action_env=SAUCE_READY_FILE=$SAUCE_READY_FILE \
--action_env=SAUCE_PID_FILE=$SAUCE_PID_FILE \
--action_env=SAUCE_TUNNEL_IDENTIFIER=$SAUCE_TUNNEL_IDENTIFIER"
)
BAZEL_EXIT_CODE=$?
echo "Exit code for bazel command was: $BAZEL_EXIT_CODE"
# Reenable immediate exit for failure exit code
set -e
##############################
# Close the saucelabs tunnel #
##############################
if [[ ! -f ${SAUCE_PID_FILE} ]]; then
echo "Could not find Saucelabs tunnel PID file. Cannot stop tunnel.."
exit 1
fi
echo "Shutting down Sauce Connect tunnel"
# The process id for the sauce-connect instance is stored inside of the pidfile.
tunnelProcessId=$(cat ${SAUCE_PID_FILE})
# Kill the process by using the PID that has been read from the pidfile. Note that
# we cannot use killall because CircleCI base container images don't have it installed.
kill ${tunnelProcessId}
while (ps -p ${tunnelProcessId} &> /dev/null); do
printf "."
sleep .5
done
echo ""
echo "Sauce Connect tunnel has been shut down"
exit $BAZEL_EXIT_CODE

View File

@ -1,35 +0,0 @@
#!/usr/bin/env bash
set -x -u -e -o pipefail
readonly currentDir=$(cd $(dirname $0); pwd)
# Command arguments that will be passed to sauce-connect. By default we disable SSL bumping for
# all requests. This is because SSL bumping is not needed for our test setup and in order
# to perform the SSL bumping, Saucelabs intercepts all HTTP requests in the tunnel VM and modifies
# them. This can cause flakiness as it makes all requests dependent on the SSL bumping middleware.
# See: https://wiki.saucelabs.com/display/DOCS/Troubleshooting+Sauce+Connect#TroubleshootingSauceConnect-DisablingSSLBumping
sauceArgs="--no-ssl-bump-domains all"
if [[ ! -z "${SAUCE_LOG_FILE:-}" ]]; then
mkdir -p $(dirname ${SAUCE_LOG_FILE})
sauceArgs="${sauceArgs} --logfile ${SAUCE_LOG_FILE}"
fi
if [[ ! -z "${SAUCE_READY_FILE:-}" ]]; then
mkdir -p $(dirname ${SAUCE_READY_FILE})
sauceArgs="${sauceArgs} --readyfile ${SAUCE_READY_FILE}"
fi
if [[ ! -z "${SAUCE_PID_FILE:-}" ]]; then
mkdir -p $(dirname ${SAUCE_PID_FILE})
sauceArgs="${sauceArgs} --pidfile ${SAUCE_PID_FILE}"
fi
if [[ ! -z "${SAUCE_TUNNEL_IDENTIFIER:-}" ]]; then
sauceArgs="${sauceArgs} --tunnel-identifier ${SAUCE_TUNNEL_IDENTIFIER}"
fi
echo "Starting Sauce Connect. Passed arguments: ${sauceArgs}"
${currentDir}/../../node_modules/sauce-connect/bin/sc -u ${SAUCE_USERNAME} -k ${SAUCE_ACCESS_KEY} ${sauceArgs}

View File

@ -1,27 +0,0 @@
#!/usr/bin/env bash
# Disable printing of any executed command because this would cause a lot
# of spam due to the loop.
set +x -u -e -o pipefail
if [[ ! -f ${SAUCE_PID_FILE} ]]; then
echo "Could not find Saucelabs tunnel PID file. Cannot stop tunnel.."
exit 1
fi
echo "Shutting down Sauce Connect tunnel"
# The process id for the sauce-connect instance is stored inside of the pidfile.
tunnelProcessId=$(cat ${SAUCE_PID_FILE})
# Kill the process by using the PID that has been read from the pidfile. Note that
# we cannot use killall because CircleCI base container images don't have it installed.
kill ${tunnelProcessId}
while (ps -p ${tunnelProcessId} &> /dev/null); do
printf "."
sleep .5
done
echo ""
echo "Sauce Connect tunnel has been shut down"

View File

@ -1,28 +0,0 @@
#!/usr/bin/env bash
# Disable printing of any executed command because this would cause a lot
# of spam due to the loop.
set +x -u -e -o pipefail
# Waits for Saucelabs Connect to be ready before executing any tests.
counter=0
while [[ ! -f ${SAUCE_READY_FILE} ]]; do
counter=$((counter + 1))
# Counter needs to be multiplied by two because the while loop only sleeps a half second.
# This has been made in favor of better progress logging (printing dots every half second)
if [ $counter -gt $[${SAUCE_READY_FILE_TIMEOUT} * 2] ]; then
echo "Timed out after ${SAUCE_READY_FILE_TIMEOUT} seconds waiting for tunnel ready file."
echo "Printing logfile output:"
echo ""
cat ${SAUCE_LOG_FILE}
exit 5
fi
printf "."
sleep 0.5
done
echo ""
echo "Connected to Saucelabs"

View File

@ -0,0 +1,50 @@
load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary")
package(default_visibility = ["//visibility:public"])
sh_binary(
name = "sauce_service_setup",
srcs = ["sauce-service.sh"],
args = ["setup"],
data = ["@npm//sauce-connect"],
)
sh_binary(
name = "sauce_service_start",
srcs = ["sauce-service.sh"],
args = ["start"],
data = ["@npm//sauce-connect"],
)
sh_binary(
name = "sauce_service_start_ready_wait",
srcs = ["sauce-service.sh"],
args = ["start-ready-wait"],
data = ["@npm//sauce-connect"],
)
sh_binary(
name = "sauce_service_ready_wait",
srcs = ["sauce-service.sh"],
args = ["ready-wait"],
data = ["@npm//sauce-connect"],
)
sh_binary(
name = "sauce_service_stop",
srcs = ["sauce-service.sh"],
args = ["stop"],
data = ["@npm//sauce-connect"],
)
nodejs_binary(
name = "karma-saucelabs",
data = [
"sauce-service.sh",
"@npm//karma",
"@npm//sauce-connect",
"@npm//shelljs",
],
entry_point = "karma-saucelabs.js",
templated_args = ["$(location sauce-service.sh)"],
)

51
tools/saucelabs/README.md Normal file
View File

@ -0,0 +1,51 @@
# Saucelabs testing with Bazel
## Local testing
Setup your `SAUCE_USERNAME`, `SAUCE_ACCESS_KEY` & `SAUCE_TUNNEL_IDENTIFIER` environment variables. These are required. On OSX, also set `SAUCE_CONNECT` to the path of your `sc` binary.
To run tests use:
```
yarn bazel run //tools/saucelabs:sauce_service_setup
yarn bazel test //path/to:saucelabs_test_target_1 --config=saucelabs [--config=ivy]
yarn bazel test //path/to:saucelabs_test_target_2 --config=saucelabs [--config=ivy]
```
or if the tests are combined into a test suite:
```
yarn bazel run //tools/saucelabs:sauce_service_setup
yarn bazel test //path/to:saucelabs_test_suite --config=saucelabs [--config=ivy]
```
To see the test output while the tests are running as these are long tests, add the `--test_output=streamed` option. Note, this option will also prevent bazel from using the test cache and will force the test to run.
The `//tools/saucelabs:sauce_service_setup` target does not start the Sauce Connect proxy but it does start process which will that then listens for the start signal from the service manager script. This signal is sent by the karma wrapper script `//tools/saucelabs:karma-saucelabs` which calls `./tools/saucelabs/sauce-service.sh start`. This is necessary as the Sauce Connect Proxy process must be started outside of `bazel test` as Bazel will automatically kill any processes spwaned during a test when that tests completes, which would prevent the tunnel from being shared by multiple tests.
# Under the hood
The karma_web_test rule is to test with saucelabs with a modified `karma` attribute set to
`//tools/saucelabs:karma-saucelabs`. This runs the `/tools/saucelabs/karma-saucelabs.js` wrapper
script which configures the saucelabs environment and starts Sauce Connect before running karma.
For example,
```
karma_web_test(
name = "saucelabs_core_acceptance_tests",
timeout = "long",
karma = "//tools/saucelabs:karma-saucelabs",
tags = [
"manual",
"no-remote-exec",
],
deps = [
"//packages/core/test/acceptance:acceptance_lib",
],
)
```
These saucelabs targets must be tagged `no-remote-exec` as they cannot be executed remotely since
they require a local Sauce Connect process. They should also be tagged `manual` so they are not
automatically tested with `//...`.

View File

@ -0,0 +1,62 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
'use strict';
const shell = require('shelljs');
const karmaBin = require.resolve('karma/bin/karma');
const runfiles = require(process.env['BAZEL_NODE_RUNFILES_HELPER']);
const sauceService = runfiles.resolve(process.argv[2]);
process.argv = [
process.argv[0],
karmaBin,
...process.argv.splice(3),
];
try {
console.error(`Setting up environment for SauceLabs karma tests...`);
// KARMA_WEB_TEST_MODE is set which informs /karma-js.conf.js that it should
// run the test with the karma saucelabs launcher
process.env['KARMA_WEB_TEST_MODE'] = 'SL_REQUIRED';
// Setup required SAUCE_* env if they are not already set
if (!process.env['SAUCE_USERNAME'] || !process.env['SAUCE_ACCESS_KEY'] ||
!process.env['SAUCE_TUNNEL_IDENTIFIER']) {
try {
// The following path comes from /tools/saucelabs/sauce-service.sh.
// We setup the required saucelabs environment variables here for the karma test
// from a json file under /tmp/angular/sauce-service so that we don't break the
// test cache with a changing SAUCE_TUNNEL_IDENTIFIER provided through --test_env
const scParams = require('/tmp/angular/sauce-service/sauce-connect-params.json');
process.env['SAUCE_USERNAME'] = scParams.SAUCE_USERNAME;
process.env['SAUCE_ACCESS_KEY'] = scParams.SAUCE_ACCESS_KEY;
process.env['SAUCE_TUNNEL_IDENTIFIER'] = scParams.SAUCE_TUNNEL_IDENTIFIER;
} catch (e) {
console.error(e.stack || e);
console.error(
`!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!! Make sure that you have run "yarn bazel run //tools/saucelabs:sauce_service_setup"
!!! (or "./tools/saucelabs/sauce-service.sh setup") before the test target. Alternately
!!! you can provide the required SAUCE_* environment variables (SAUCE_USERNAME, SAUCE_ACCESS_KEY &
!!! SAUCE_TUNNEL_IDENTIFIER) to the test with --test_env or --define but this may prevent bazel from
!!! using cached test results.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
process.exit(1);
}
}
const scStart = `${sauceService} start-ready-wait`;
console.error(`Starting SauceConnect (${scStart})...`);
const result = shell.exec(scStart).code;
if (result !== 0) {
throw new Error(`Starting SauceConnect failed with code ${result}`);
}
console.error(`Launching karma ${karmaBin}...`);
module.constructor._load(karmaBin, this, /*isMain=*/true);
} catch (e) {
console.error(e.stack || e);
process.exit(1);
}

419
tools/saucelabs/sauce-service.sh Executable file
View File

@ -0,0 +1,419 @@
#!/usr/bin/env bash
set -u -e -o pipefail
####################################################################################################
# Some helper funtions
@echo() {
echo "# $*"
}
@warn() {
@echo "Warning: $*" >&2
}
@fail() {
@echo "Error! $*" >&2
exit 1
}
@remove() {
local f="$1"
if [[ -f ${f} ]]; then
@echo "Removing ${f}"
rm -f "${f}" || @fail "Can not delete ${f} file"
fi
}
@kill() {
for p in $1; do
if kill -0 ${p} >/dev/null 2>&1; then
kill ${p}
sleep 2
if kill -0 ${p} >/dev/null 2>&1; then
kill -9 ${p}
sleep 2
fi
fi
done
}
@wait_for() {
local m="$1"
local f="$2"
if [[ ! -f "${f}" ]]; then
printf "# ${m} (${f})"
while [[ ! -f "${f}" ]]; do
printf "."
sleep 0.5
done
printf "\n"
fi
}
####################################################################################################
# Sauce service functions
readonly SCRIPT_DIR=$(cd $(dirname $0); pwd)
readonly TMP_DIR="/tmp/angular/sauce-service"
mkdir -p ${TMP_DIR}
# Location for the saucelabs log file.
readonly SAUCE_LOG_FILE="${TMP_DIR}/sauce-connect.log"
# Location for the saucelabs ready to connection process id lock file.
readonly SAUCE_PID_FILE="${TMP_DIR}/sauce-connect.pid"
# Location for the saucelabs ready to connect lock file.
readonly SAUCE_READY_FILE="${TMP_DIR}/sauce-connect.lock"
# Location for the saucelabs params file for use by test runner.
readonly SAUCE_PARAMS_JSON_FILE="${TMP_DIR}/sauce-connect-params.json"
# Amount of seconds we wait for sauceconnect to establish a tunnel instance. In order to not
# acquire CircleCI instances for too long if sauceconnect fails, we need a connect timeout.
readonly SAUCE_READY_FILE_TIMEOUT=120
readonly SERVICE_LOCK_FILE="${TMP_DIR}/service.lock"
readonly SERVICE_START_FILE="${TMP_DIR}/service.start"
readonly SERVICE_PID_FILE="${TMP_DIR}/service.pid"
readonly SERVICE_LOG_FILE="${TMP_DIR}/service.log"
service-setup-command() {
if [[ -z "${SAUCE_USERNAME:-}" ]]; then
@fail "SAUCE_USERNAME environment variable required"
fi
if [[ -z "${SAUCE_ACCESS_KEY:-}" ]]; then
@fail "SAUCE_ACCESS_KEY environment variable required"
fi
if [[ -z "${SAUCE_TUNNEL_IDENTIFIER:-}" ]]; then
@fail "SAUCE_TUNNEL_IDENTIFIER environment variable required"
fi
local unameOut="$(uname -s)"
case "${unameOut}" in
Linux*) local machine=linux ;;
Darwin*) local machine=darwin ;;
CYGWIN*) local machine=windows ;;
MINGW*) local machine=windows ;;
MSYS_NT*) local machine=windows ;;
*) local machine=linux
printf "\nUnrecongized uname '${unameOut}'; defaulting to use node for linux.\n" >&2
printf "Please file an issue to https://github.com/bazelbuild/rules_nodejs/issues if \n" >&2
printf "you would like to add your platform to the supported rules_nodejs node platforms.\n\n" >&2
;;
esac
case "${machine}" in
# Path to sauce connect executable
linux)
if [[ -z "${BUILD_WORKSPACE_DIRECTORY:-}" ]]; then
# Started manually
SAUCE_CONNECT="${SCRIPT_DIR}/../../node_modules/sauce-connect/bin/sc"
else
# Started via `bazel run`
SAUCE_CONNECT="${BUILD_WORKSPACE_DIRECTORY}/node_modules/sauce-connect/bin/sc"
fi
;;
*)
if [[ -z "${SAUCE_CONNECT:-}" ]]; then
@fail "SAUCE_CONNECT environment variable is required on non-linux environments"
exit 1
fi
;;
esac
if [[ ! -f ${SAUCE_CONNECT} ]]; then
@fail "sc binary not found at ${SAUCE_CONNECT}"
fi
echo "{ \"SAUCE_USERNAME\": \"${SAUCE_USERNAME}\", \"SAUCE_ACCESS_KEY\": \"${SAUCE_ACCESS_KEY}\", \"SAUCE_TUNNEL_IDENTIFIER\": \"${SAUCE_TUNNEL_IDENTIFIER}\" }" > ${SAUCE_PARAMS_JSON_FILE}
# Command arguments that will be passed to sauce-connect.
# By default we disable SSL bumping for all requests. This is because SSL bumping is
# not needed for our test setup and in order to perform the SSL bumping, Saucelabs
# intercepts all HTTP requests in the tunnel VM and modifies them. This can cause
# flakiness as it makes all requests dependent on the SSL bumping middleware.
# See: https://wiki.saucelabs.com/display/DOCS/Troubleshooting+Sauce+Connect#TroubleshootingSauceConnect-DisablingSSLBumping
local sauce_args=(
"--no-ssl-bump-domains all"
"--logfile ${SAUCE_LOG_FILE}"
"--pidfile ${SAUCE_PID_FILE}"
"--readyfile ${SAUCE_READY_FILE}"
"--tunnel-identifier ${SAUCE_TUNNEL_IDENTIFIER}"
"--user ${SAUCE_USERNAME}"
# Don't add the --api-key here so we don't echo it out in service-pre-start
)
@echo "Sauce connect will be started with:"
echo " ${SAUCE_CONNECT} ${sauce_args[@]}"
SERVICE_COMMAND="${SAUCE_CONNECT} ${sauce_args[@]} --api-key ${SAUCE_ACCESS_KEY}"
}
# Called by pre-start & post-stop
service-cleanup() {
if [[ -f "${SAUCE_PID_FILE}" ]]; then
local p=$(cat "${SAUCE_PID_FILE}")
@echo "Stopping Sauce Connect (pid $p)..."
@kill $p
fi
@remove "${SAUCE_PID_FILE}"
@remove "${SAUCE_READY_FILE}"
@remove "${SAUCE_PARAMS_JSON_FILE}"
}
# Called before service is setup
service-pre-setup() {
service-cleanup
}
# Called after service is setup
service-post-setup() {
@echo " sauce params : ${SAUCE_PARAMS_JSON_FILE}"
}
# Called before service is started
service-pre-start() {
return
}
# Called after service is started
service-post-start() {
@wait_for "Waiting for Sauce Connect Proxy process" "${SAUCE_PID_FILE}"
@echo "Sauce Connect Proxy started (pid $(cat "${SAUCE_PID_FILE}"))"
}
# Called if service fails to start
service-failed-setup() {
if [[ -f "${SERVICE_LOG_FILE}" ]]; then
echo "================================================================================"
echo "${SERVICE_LOG_FILE}:"
echo $(cat "${SERVICE_LOG_FILE}")
fi
}
# Called by ready-wait action
service-ready-wait() {
if [[ ! -f "${SAUCE_PID_FILE}" ]]; then
@fail "Sauce Connect not running"
fi
if [[ ! -f "${SAUCE_READY_FILE}" ]]; then
# Wait for saucelabs tunnel to connect
printf "# Waiting for saucelabs tunnel to connect (${SAUCE_READY_FILE})"
counter=0
while [[ ! -f "${SAUCE_READY_FILE}" ]]; do
counter=$((counter + 1))
# Counter needs to be multiplied by two because the while loop only sleeps a half second.
# This has been made in favor of better progress logging (printing dots every half second)
if [ $counter -gt $[${SAUCE_READY_FILE_TIMEOUT} * 2] ]; then
@echo "Timed out after ${SAUCE_READY_FILE_TIMEOUT} seconds waiting for tunnel ready file."
if [[ -f "${SAUCE_LOG_FILE}" ]]; then
echo "================================================================================"
echo "${SAUCE_LOG_FILE}:"
cat "${SAUCE_LOG_FILE}"
fi
exit 5
fi
printf "."
sleep 0.5
done
printf "\n"
@echo "Saucelabs tunnel connected"
else
@echo "Saucelabs tunnel already connected"
fi
}
# Called before service is stopped
service-pre-stop() {
return
}
# Called after service is stopped
service-post-stop() {
service-cleanup
}
####################################################################################################
# Generic service functions
# This uses functions setup above but nothing below should be specific to saucelabs
@serviceLock() {
# Check is Lock File exists, if not create it and set trap on exit
printf "# Waiting for service action lock (${SERVICE_LOCK_FILE})"
while true; do
if { set -C; 2>/dev/null >"${SERVICE_LOCK_FILE}"; }; then
trap "rm -f \"${SERVICE_LOCK_FILE}\"" EXIT
printf "\n"
break
fi
printf "."
sleep 0.5
done
@echo "Acquired service action lock"
}
@serviceStatus() {
if [ -f "${SERVICE_PID_FILE}" ] && [ ! -z "$(cat "${SERVICE_PID_FILE}")" ]; then
local p=$(cat "${SERVICE_PID_FILE}")
if kill -0 $p >/dev/null 2>&1; then
@echo "Service is running (pid $p)"
return 0
else
@echo "Service is not running (process PID $p not exists)"
return 1
fi
else
@echo "Service is not running"
return 2
fi
}
@serviceSetup() {
if @serviceStatus >/dev/null 2>&1; then
@echo "Service already running (pid $(cat "${SERVICE_PID_FILE}"))"
return 0
fi
@echo "Setting up service..."
@remove "${SERVICE_PID_FILE}"
@remove "${SERVICE_START_FILE}"
touch "${SERVICE_LOG_FILE}" >/dev/null 2>&1 || @fail "Can not create ${SERVICE_LOG_FILE} file"
@echo " service pid : ${SERVICE_PID_FILE}"
@echo " service logs : ${SERVICE_LOG_FILE}"
service-pre-setup
service-setup-command
(
(
if [[ -z "${SERVICE_COMMAND:-}" ]]; then
@fail "No SERVICE_COMMAND is set"
fi
@wait_for "Waiting for start file" "${SERVICE_START_FILE}"
${SERVICE_COMMAND}
) >>"${SERVICE_LOG_FILE}" 2>&1
) &
echo $! >"${SERVICE_PID_FILE}"
if @serviceStatus >/dev/null 2>&1; then
@echo "Service setup (pid $(cat "${SERVICE_PID_FILE}"))"
service-post-setup
else
@echo "Error setting up Service!"
service-failed-setup
exit 1
fi
return $?
}
@serviceStart() {
if @serviceStatus >/dev/null 2>&1; then
@echo "Service already setup (pid $(cat "${SERVICE_PID_FILE}"))"
else
@serviceSetup
fi
if [[ -f "${SERVICE_START_FILE}" ]]; then
@echo "Service already started"
else
@echo "Starting service..."
service-pre-start
touch "${SERVICE_START_FILE}" >/dev/null 2>&1 || @err "Can not create ${SERVICE_START_FILE} file"
service-post-start
@echo "Service started"
fi
}
@serviceStop() {
if @serviceStatus >/dev/null 2>&1; then
touch "${SERVICE_PID_FILE}" >/dev/null 2>&1 || @fail "Can not touch ${SERVICE_PID_FILE} file"
service-pre-stop
@echo "Stopping sevice (pid $(cat "${SERVICE_PID_FILE}"))..."
@kill $(cat "${SERVICE_PID_FILE}")
if @serviceStatus >/dev/null 2>&1; then
@fail "Error stopping Service! Service already running with PID $(cat "${SERVICE_PID_FILE}")"
else
@echo "Service stopped"
@remove "${SERVICE_PID_FILE}"
@remove "${SERVICE_START_FILE}"
service-post-stop
fi
return 0
else
@warn "Service is not running"
service-post-stop
fi
}
@serviceStartReadyWait() {
@serviceStart
@serviceReadyWait
}
@serviceReadyWait() {
service-ready-wait
}
@serviceRestart() {
@serviceStop
@serviceStart
}
@serviceTail() {
tail -f "${SERVICE_LOG_FILE}"
}
case "${1:-}" in
setup)
@serviceLock
@serviceSetup
;;
start)
@serviceLock
@serviceStart
;;
start-ready-wait)
@serviceLock
@serviceStartReadyWait
;;
ready-wait)
@serviceLock
@serviceReadyWait
;;
stop)
@serviceLock
@serviceStop
;;
restart)
@serviceLock
@serviceRestart
;;
status)
@serviceLock
@serviceStatus
;;
run)
(
service-setup-command
if [[ -z "${SERVICE_COMMAND:-}" ]]; then
@fail "No SERVICE_COMMAND is set"
fi
${SERVICE_COMMAND}
)
;;
tail)
@serviceTail
;;
*)
@echo "Actions: [setup|start|start-read-wait|ready-wait|stop|restart|status|run|tail]"
exit 1
;;
esac