feat(bazel): use rbe_autoconfig() and new container. (#29336)

After this PR is merged, maintainers no longer need to update .bazelrc
file, toolchain and platform related flags for RBE builds and tests
(unless there is a breaking change in Bazel related to those flags).

Maintainers just need to update the pin of @bazel-toolchains repo
regularly in the packages/bazel/package.bzl file according to
https://releases.bazel.build/bazel-toolchains.html to include the
latest checked-in toolchain configs. If rbe_autoconfig() cannot find
appropriate toolchain configs for the version of Bazel in the version of
@bazel_toolchains repo that is currently used by this project, it will pull
down the container and generate the configs on the fly as the beginning
of the build/test.

PR Close #29336
This commit is contained in:
Xin Gao 2019-03-15 13:45:20 -04:00 committed by Alex Rickabaugh
parent 8d3365e4fc
commit 9abf114fbb
8 changed files with 93 additions and 263 deletions

View File

@ -116,16 +116,21 @@ build --define=compile=legacy
# --config=remote # --config=remote
############################### ###############################
# Load default settings for Remote Build Execution # Load default settings for Remote Build Execution.
# When updating, the URLs of bazel_toolchains in packages/bazel/package.bzl import %workspace%/third_party/github.com/bazelbuild/bazel-toolchains/bazelrc/.bazelrc.notoolchain
# may also need to be updated (see https://github.com/angular/angular/pull/27935)
import %workspace%/third_party/github.com/bazelbuild/bazel-toolchains/bazelrc/bazel-0.24.0.bazelrc
# Increase the default number of jobs by 50% because our build has lots of # Increase the default number of jobs by 50% because our build has lots of
# parallelism # parallelism
build:remote --jobs=150 build:remote --jobs=150
# Point to our custom execution platform; see tools/BUILD.bazel # Toolchain and platform related flags
build:remote --host_javabase=@rbe_ubuntu1604_angular//java:jdk
build:remote --javabase=@rbe_ubuntu1604_angular//java:jdk
build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
build:remote --crosstool_top=@rbe_ubuntu1604_angular//cc:toolchain
build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
build:remote --extra_toolchains=@rbe_ubuntu1604_angular//config:cc-toolchain
build:remote --extra_execution_platforms=//tools:rbe_ubuntu1604-angular build:remote --extra_execution_platforms=//tools:rbe_ubuntu1604-angular
build:remote --host_platform=//tools:rbe_ubuntu1604-angular build:remote --host_platform=//tools:rbe_ubuntu1604-angular
build:remote --platforms=//tools:rbe_ubuntu1604-angular build:remote --platforms=//tools:rbe_ubuntu1604-angular

View File

@ -167,7 +167,8 @@ jobs:
- *attach_workspace - *attach_workspace
- *init_environment - *init_environment
- *setup_circleci_bazel_config - *setup_circleci_bazel_config
# Enable remote/sibling docker which is needed by auto-selection of toolchain configs for RBE.
- setup_remote_docker
# Setup remote execution and run RBE-compatible tests. # Setup remote execution and run RBE-compatible tests.
- *setup_bazel_remote_execution - *setup_bazel_remote_execution
- run: yarn bazel test //... --build_tag_filters=-ivy-only --test_tag_filters=-ivy-only - run: yarn bazel test //... --build_tag_filters=-ivy-only --test_tag_filters=-ivy-only
@ -180,6 +181,8 @@ jobs:
- *attach_workspace - *attach_workspace
- *init_environment - *init_environment
- *setup_circleci_bazel_config - *setup_circleci_bazel_config
# Enable remote/sibling docker which is needed by auto-selection of toolchain configs for RBE.
- setup_remote_docker
- *setup_bazel_remote_execution - *setup_bazel_remote_execution
# We need to explicitly specify the --symlink_prefix option because otherwise we would # We need to explicitly specify the --symlink_prefix option because otherwise we would
@ -409,6 +412,8 @@ jobs:
- *attach_workspace - *attach_workspace
- *init_environment - *init_environment
- *setup_circleci_bazel_config - *setup_circleci_bazel_config
# Enable remote/sibling docker which is needed by auto-selection of toolchain configs for RBE.
- setup_remote_docker
- *setup_bazel_remote_execution - *setup_bazel_remote_execution
- run: scripts/build-packages-dist.sh - run: scripts/build-packages-dist.sh
@ -435,6 +440,8 @@ jobs:
- *attach_workspace - *attach_workspace
- *init_environment - *init_environment
- *setup_circleci_bazel_config - *setup_circleci_bazel_config
# Enable remote/sibling docker which is needed by auto-selection of toolchain configs for RBE.
- setup_remote_docker
- *setup_bazel_remote_execution - *setup_bazel_remote_execution
- run: scripts/build-ivy-npm-packages.sh - run: scripts/build-ivy-npm-packages.sh

View File

@ -114,3 +114,23 @@ sass_repositories()
load("@io_bazel_skydoc//skylark:skylark.bzl", "skydoc_repositories") load("@io_bazel_skydoc//skylark:skylark.bzl", "skydoc_repositories")
skydoc_repositories() skydoc_repositories()
load("@bazel_toolchains//rules:environments.bzl", "clang_env")
load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig")
rbe_autoconfig(
name = "rbe_ubuntu1604_angular",
# The sha256 of marketplace.gcr.io/google/rbe-ubuntu16-04 container that is
# used by rbe_autoconfig() to pair toolchain configs in the @bazel_toolchains repo.
base_container_digest = "sha256:677c1317f14c6fd5eba2fd8ec645bfdc5119f64b3e5e944e13c89e0525cc8ad1",
# Note that if you change the `digest`, you might also need to update the
# `base_container_digest` to make sure marketplace.gcr.io/google/rbe-ubuntu16-04-webtest:<digest>
# and marketplace.gcr.io/google/rbe-ubuntu16-04:<base_container_digest> have
# the same Clang and JDK installed.
# Clang is needed because of the dependency on @com_google_protobuf.
# Java is needed for the Bazel's test executor Java tool.
digest = "sha256:74a8e9dca4781d5f277a7bd8e7ea7ed0f5906c79c9cd996205b6d32f090c62f3",
env = clang_env(),
registry = "marketplace.gcr.io",
repository = "google/rbe-ubuntu16-04-webtest",
)

View File

@ -28,11 +28,10 @@ def rules_angular_dev_dependencies():
_maybe( _maybe(
http_archive, http_archive,
name = "bazel_toolchains", name = "bazel_toolchains",
sha256 = "67335b3563d9b67dc2550b8f27cc689b64fadac491e69ce78763d9ba894cc5cc", sha256 = "142bcbd8cb751ce1193a1d7fef4e328493cd0a69cc0555183ad237f81418ba40",
strip_prefix = "bazel-toolchains-cddc376d428ada2927ad359211c3e356bd9c9fbb", strip_prefix = "bazel-toolchains-628224f6cf48e81116d0ee0bf65424eaa630d5b3",
urls = [ urls = [
"https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/cddc376d428ada2927ad359211c3e356bd9c9fbb.tar.gz", "https://github.com/xingao267/bazel-toolchains/archive/628224f6cf48e81116d0ee0bf65424eaa630d5b3.tar.gz",
"https://github.com/bazelbuild/bazel-toolchains/archive/cddc376d428ada2927ad359211c3e356bd9c9fbb.tar.gz",
], ],
) )

View File

@ -0,0 +1,50 @@
# Copyright 2016 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This .bazelrc file contains some of the flags required for the provided
# toolchain with Remote Build Execution. Specifically, it includes all flags
# except toolchain/platform flags.
# Depending on how many machines are in the remote execution instance, setting
# this higher can make builds faster by allowing more jobs to run in parallel.
# Setting it too high can result in jobs that timeout, however, while waiting
# for a remote machine to execute them.
build:remote --jobs=50
# Set various strategies so that all actions execute remotely. Mixing remote
# and local execution will lead to errors unless the toolchain and remote
# machine exactly match the host machine.
build:remote --spawn_strategy=remote
build:remote --strategy=Javac=remote
build:remote --strategy=Closure=remote
build:remote --strategy=Genrule=remote
build:remote --define=EXECUTOR=remote
# Enable the remote cache so action results can be shared across machines,
# developers, and workspaces.
build:remote --remote_cache=remotebuildexecution.googleapis.com
# Enable remote execution so actions are performed on the remote systems.
build:remote --remote_executor=remotebuildexecution.googleapis.com
# Enable encryption.
build:remote --tls_enabled=true
# Set a higher timeout value, just in case.
build:remote --remote_timeout=3600
# Enable authentication. This will pick up application default credentials by
# default. You can use --auth_credentials=some_file.json to use a service
# account credential instead.
build:remote --auth_enabled=true

View File

@ -1,127 +0,0 @@
# Copyright 2016 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is auto-generated from release/bazelrc.tpl and should not be
# modified directly.
# This .bazelrc file contains all of the flags required for the toolchain,
# Remote Build Execution, and the Bazel Build Results UI. Specific flags in
# your Bazel command allow you to use only the remote build, to use only the
# results UI, or to use them both together.
#
# This .bazelrc file also contains all of the flags required for the local
# docker sandboxing.
# Depending on how many machines are in the remote execution instance, setting
# this higher can make builds faster by allowing more jobs to run in parallel.
# Setting it too high can result in jobs that timeout, however, while waiting
# for a remote machine to execute them.
build:remote --jobs=50
# Set several flags related to specifying the platform, toolchain and java
# properties.
# These flags are duplicated rather than imported from (for example)
# %workspace%/configs/ubuntu16_04_clang/1.1/toolchain.bazelrc to make this
# bazelrc a standalone file that can be copied more easily.
# These flags should only be used as is for the rbe-ubuntu16-04 container
# and need to be adapted to work with other toolchain containers.
build:remote --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8
build:remote --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8
build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
build:remote --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.1/bazel_0.21.0/default:toolchain
build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
# Platform flags:
# The toolchain container used for execution is defined in the target indicated
# by "extra_execution_platforms", "host_platform" and "platforms".
# If you are using your own toolchain container, you need to create a platform
# target with "constraint_values" that allow for the toolchain specified with
# "extra_toolchains" to be selected (given constraints defined in
# "exec_compatible_with").
# More about platforms: https://docs.bazel.build/versions/master/platforms.html
build:remote --extra_toolchains=@bazel_toolchains//configs/ubuntu16_04_clang/1.1/bazel_0.21.0/cpp:cc-toolchain-clang-x86_64-default
build:remote --extra_execution_platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:rbe_ubuntu1604
build:remote --host_platform=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:rbe_ubuntu1604
build:remote --platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:rbe_ubuntu1604
# Set various strategies so that all actions execute remotely. Mixing remote
# and local execution will lead to errors unless the toolchain and remote
# machine exactly match the host machine.
build:remote --spawn_strategy=remote
build:remote --strategy=Javac=remote
build:remote --strategy=Closure=remote
build:remote --genrule_strategy=remote
build:remote --define=EXECUTOR=remote
# Enable the remote cache so action results can be shared across machines,
# developers, and workspaces.
build:remote --remote_cache=remotebuildexecution.googleapis.com
# Enable remote execution so actions are performed on the remote systems.
build:remote --remote_executor=remotebuildexecution.googleapis.com
# Enable encryption.
build:remote --tls_enabled=true
# Set a higher timeout value, just in case.
build:remote --remote_timeout=3600
# Enable authentication. This will pick up application default credentials by
# default. You can use --auth_credentials=some_file.json to use a service
# account credential instead.
build:remote --auth_enabled=true
# Set flags for uploading to BES in order to view results in the Bazel Build
# Results UI.
build:results --bes_backend="buildeventservice.googleapis.com"
build:results --bes_timeout=60s
build:results --tls_enabled
# Output BES results url
build:results --bes_results_url="https://source.cloud.google.com/results/invocations/"
# Set flags for uploading to BES without Remote Build Execution.
build:results-local --bes_backend="buildeventservice.googleapis.com"
build:results-local --bes_timeout=60s
build:results-local --tls_enabled=true
build:results-local --auth_enabled=true
build:results-local --spawn_strategy=local
build:results-local --remote_cache=remotebuildexecution.googleapis.com
build:results-local --remote_timeout=3600
build:results-local --bes_results_url="https://source.cloud.google.com/results/invocations/"
# The following flags are only necessary for local docker sandboxing
# with the rbe-ubuntu16-04 container. Use of these flags is still experimental.
build:docker-sandbox --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8
build:docker-sandbox --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8
build:docker-sandbox --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.1/bazel_0.21.0/default:toolchain
build:docker-sandbox --experimental_docker_image=gcr.io/cloud-marketplace/google/rbe-ubuntu16-04@sha256:9bd8ba020af33edb5f11eff0af2f63b3bcb168cd6566d7b27c6685e717787928
build:docker-sandbox --spawn_strategy=docker
build:docker-sandbox --strategy=Javac=docker
build:docker-sandbox --strategy=Closure=docker
build:docker-sandbox --genrule_strategy=docker
build:docker-sandbox --define=EXECUTOR=remote
build:docker-sandbox --experimental_docker_verbose
build:docker-sandbox --experimental_enable_docker_sandbox
# The following flags enable the remote cache so action results can be shared
# across machines, developers, and workspaces.
build:remote-cache --remote_cache=remotebuildexecution.googleapis.com
build:remote-cache --tls_enabled=true
build:remote-cache --remote_timeout=3600
build:remote-cache --auth_enabled=true
build:remote-cache --spawn_strategy=standalone
build:remote-cache --strategy=Javac=standalone
build:remote-cache --strategy=Closure=standalone
build:remote-cache --genrule_strategy=standalone

View File

@ -1,106 +0,0 @@
# Copyright 2016 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is auto-generated from release/bazelrc.tpl and should not be
# modified directly.
# This .bazelrc file contains all of the flags required for the provided
# toolchain with Remote Build Execution.
#
# This .bazelrc file also contains all of the flags required for the local
# docker sandboxing.
# Depending on how many machines are in the remote execution instance, setting
# this higher can make builds faster by allowing more jobs to run in parallel.
# Setting it too high can result in jobs that timeout, however, while waiting
# for a remote machine to execute them.
build:remote --jobs=50
# Set several flags related to specifying the platform, toolchain and java
# properties.
# These flags are duplicated rather than imported from (for example)
# %workspace%/configs/ubuntu16_04_clang/1.2/toolchain.bazelrc to make this
# bazelrc a standalone file that can be copied more easily.
# These flags should only be used as is for the rbe-ubuntu16-04 container
# and need to be adapted to work with other toolchain containers.
build:remote --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8
build:remote --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8
build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
build:remote --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.2/bazel_0.24.0/default:toolchain
build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
# Platform flags:
# The toolchain container used for execution is defined in the target indicated
# by "extra_execution_platforms", "host_platform" and "platforms".
# If you are using your own toolchain container, you need to create a platform
# target with "constraint_values" that allow for the toolchain specified with
# "extra_toolchains" to be selected (given constraints defined in
# "exec_compatible_with").
# More about platforms: https://docs.bazel.build/versions/master/platforms.html
build:remote --extra_toolchains=@bazel_toolchains//configs/ubuntu16_04_clang/1.2/bazel_0.24.0/cpp:cc-toolchain-clang-x86_64-default
build:remote --extra_execution_platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:rbe_ubuntu1604
build:remote --host_platform=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:rbe_ubuntu1604
build:remote --platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:rbe_ubuntu1604
# Set various strategies so that all actions execute remotely. Mixing remote
# and local execution will lead to errors unless the toolchain and remote
# machine exactly match the host machine.
build:remote --spawn_strategy=remote
build:remote --strategy=Javac=remote
build:remote --strategy=Closure=remote
build:remote --strategy=Genrule=remote
build:remote --define=EXECUTOR=remote
# Enable the remote cache so action results can be shared across machines,
# developers, and workspaces.
build:remote --remote_cache=remotebuildexecution.googleapis.com
# Enable remote execution so actions are performed on the remote systems.
build:remote --remote_executor=remotebuildexecution.googleapis.com
# Enable encryption.
build:remote --tls_enabled=true
# Set a higher timeout value, just in case.
build:remote --remote_timeout=3600
# Enable authentication. This will pick up application default credentials by
# default. You can use --auth_credentials=some_file.json to use a service
# account credential instead.
build:remote --auth_enabled=true
# The following flags are only necessary for local docker sandboxing
# with the rbe-ubuntu16-04 container. Use of these flags is still experimental.
build:docker-sandbox --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8
build:docker-sandbox --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8
build:docker-sandbox --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.2/bazel_0.24.0/default:toolchain
build:docker-sandbox --experimental_docker_image=gcr.io/cloud-marketplace/google/rbe-ubuntu16-04@sha256:da0f21c71abce3bbb92c3a0c44c3737f007a82b60f8bd2930abc55fe64fc2729
build:docker-sandbox --spawn_strategy=docker
build:docker-sandbox --strategy=Javac=docker
build:docker-sandbox --strategy=Closure=docker
build:docker-sandbox --strategy=Genrule=docker
build:docker-sandbox --define=EXECUTOR=remote
build:docker-sandbox --experimental_docker_verbose
build:docker-sandbox --experimental_enable_docker_sandbox
# The following flags enable the remote cache so action results can be shared
# across machines, developers, and workspaces.
build:remote-cache --remote_cache=remotebuildexecution.googleapis.com
build:remote-cache --tls_enabled=true
build:remote-cache --remote_timeout=3600
build:remote-cache --auth_enabled=true
build:remote-cache --spawn_strategy=standalone
build:remote-cache --strategy=Javac=standalone
build:remote-cache --strategy=Closure=standalone
build:remote-cache --strategy=Genrule=standalone

View File

@ -13,29 +13,11 @@ ts_config(
deps = ["tsconfig.json"], deps = ["tsconfig.json"],
) )
# The toolchain container used for execution is defined in the target indicated
# by "extra_execution_platforms", "host_platform" and "platforms".
# If you are using your own toolchain container, you need to create a platform
# target with "constraint_values" that allow for the toolchain specified with
# "extra_toolchains" to be selected (given constraints defined in
# "exec_compatible_with").
# More about platforms: https://docs.bazel.build/versions/master/platforms.html
platform( platform(
name = "rbe_ubuntu1604-angular", name = "rbe_ubuntu1604-angular",
constraint_values = [ parents = ["@rbe_ubuntu1604_angular//config:platform"],
"@bazel_tools//platforms:x86_64",
"@bazel_tools//platforms:linux",
"@bazel_tools//tools/cpp:clang",
"@bazel_toolchains//constraints:xenial",
],
# We are currently using the docker container image provided by RBE. Once
# Angular has the billing account setup, we should start using/maintaining
# our own docker container image.
remote_execution_properties = """ remote_execution_properties = """
properties: { {PARENT_REMOTE_EXECUTION_PROPERTIES}
name: "container-image"
value:"docker://gcr.io/asci-toolchain/nosla-ubuntu16_04-webtest@sha256:e874885f5e3d9ac0c0d3176e5369cb5969467dbf9ad8d42b862829cec8d84b9b"
}
properties: { properties: {
name: "dockerAddCapabilities" name: "dockerAddCapabilities"
value: "SYS_ADMIN" value: "SYS_ADMIN"