From 671f64324ba67d887fd113df90f932eeeac18f83 Mon Sep 17 00:00:00 2001 From: Filipe Silva Date: Tue, 28 May 2019 10:41:29 +0100 Subject: [PATCH] ci: remove buildkite setup (#30510) We don't use it anymore and it's pretty stable by now. PR Close #30510 --- .buildkite/Dockerfile | 42 ---------- .buildkite/README.md | 96 ---------------------- .buildkite/fix-msys64.cmd | 6 -- .buildkite/pipeline.yml | 10 --- .buildkite/provision-windows-buildkite.ps1 | 94 --------------------- 5 files changed, 248 deletions(-) delete mode 100644 .buildkite/Dockerfile delete mode 100644 .buildkite/README.md delete mode 100644 .buildkite/fix-msys64.cmd delete mode 100644 .buildkite/pipeline.yml delete mode 100644 .buildkite/provision-windows-buildkite.ps1 diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile deleted file mode 100644 index d9d13f16e6..0000000000 --- a/.buildkite/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# Heavily based on https://github.com/StefanScherer/dockerfiles-windows/ images. -# Combines the node windowsservercore image with the Bazel Prerequisites (https://docs.bazel.build/versions/master/install-windows.html). -# msys install taken from https://github.com/StefanScherer/dockerfiles-windows/issues/30 -# VS redist install taken from https://github.com/StefanScherer/dockerfiles-windows/blob/master/apache/Dockerfile -# The nanoserver image won't work because MSYS2 does not run in it https://github.com/Alexpux/MSYS2-packages/issues/1493 - -# Before building this image, you must locally build node-windows:10.13.0-windowsservercore-1803. -# Clone https://github.com/StefanScherer/dockerfiles-windows/commit/4ce7101a766b9b880ac262479dd9126b64d656cf and build using -# docker build -t node-windows:10.13.0-windowsservercore-1803 --build-arg core=microsoft/windowsservercore:1803 --build-arg target=microsoft/windowsservercore:1803 . -FROM node-windows:10.13.0-windowsservercore-1803 - -SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] - -# Install 7zip to extract msys2 -RUN Invoke-WebRequest -UseBasicParsing 'https://www.7-zip.org/a/7z1805-x64.exe' -OutFile 7z.exe -# For some reason the last letter in the destination directory is lost. So '/D=C:\\7zip0' will extract to '/D=C:\\7zip'. -RUN Start-Process -FilePath 'C:\\7z.exe' -ArgumentList '/S', '/D=C:\\7zip0' -NoNewWindow -Wait - -# Extract msys2 -RUN Invoke-WebRequest -UseBasicParsing 'http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20180531.tar.xz' -OutFile msys2.tar.xz -RUN Start-Process -FilePath 'C:\\7zip\\7z' -ArgumentList 'e', 'msys2.tar.xz' -Wait -RUN Start-Process -FilePath 'C:\\7zip\\7z' -ArgumentList 'x', 'msys2.tar', '-oC:\\' -Wait -RUN Remove-Item msys2.tar.xz -RUN Remove-Item msys2.tar -RUN Remove-Item 7z.exe -RUN Remove-Item -Recurse 7zip - -# Add MSYS2 to PATH, and set BAZEL_SH -RUN [Environment]::SetEnvironmentVariable('Path', $env:Path + ';C:\msys64\usr\bin', [System.EnvironmentVariableTarget]::Machine) -RUN [Environment]::SetEnvironmentVariable('BAZEL_SH', 'C:\msys64\usr\bin\bash.exe', [System.EnvironmentVariableTarget]::Machine) - -# Install Microsoft Visual C++ Redistributable for Visual Studio 2015 -RUN Invoke-WebRequest -UseBasicParsing 'https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x64.exe' -OutFile vc_redist.x64.exe -RUN Start-Process 'c:\\vc_redist.x64.exe' -ArgumentList '/Install', '/Passive', '/NoRestart' -NoNewWindow -Wait -RUN Remove-Item vc_redist.x64.exe - -# Add a fix for https://github.com/docker/for-win/issues/2920 as entry point to the container. -SHELL ["cmd", "/c"] -COPY "fix-msys64.cmd" "C:\\fix-msys64.cmd" -ENTRYPOINT cmd /C C:\\fix-msys64.cmd && cmd /c - -CMD ["cmd.exe"] diff --git a/.buildkite/README.md b/.buildkite/README.md deleted file mode 100644 index b7e25cd93b..0000000000 --- a/.buildkite/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# BuildKite configuration - -This folder contains configuration for the [BuildKite](https://buildkite.com) based CI checks for -this repository. - -BuildKite is a CI provider that provides build coordination and reports while we provide the -infrastructure. - -CI runs are triggered by new PRs and will show up on the GitHub checks interface, along with the -other current CI solutions. - -Currently it is only used for tests on Windows platforms. - - -## The build pipeline - -BuildKite uses a pipeline for each repository. The `pipeline.yml` file defines pipeline -[build steps](https://buildkite.com/docs/pipelines/defining-steps) for this repository. - -Run results can be seen in the GitHub checks interface and in the -[pipeline dashboard](https://buildkite.com/angular/angular). - -Although most configuration is done via `pipeline.yml`, some options are only available -in the online [pipeline settings](https://buildkite.com/angular/angular/settings). - - -## Infrastructure - -BuildKite does not provide the host machines where the builds runs, providing instead the -[BuildKite Agent](https://buildkite.com/docs/agent/v3) that should be run our own infrastructure. - - -### Agents - -This agent polls the BuildKite API for builds, runs them, and reports back the results. -Agents are the unit of concurrency: each agent can run one build at any given time. -Adding agents allows more builds to be ran at the same time. - -Individual agents can have tags, and pipeline steps can target only agents with certain tags via the -`agents` field in `pipeline.yml`. -For example: agents on Windows machines are tagged as `windows`, and the Windows specific build -steps list `windows: true` in their `agents` field. - -You can see the current agent pool, along with their tags, in the -[agents list](https://buildkite.com/organizations/angular/agents). - - -### Our host machines - -We use [Google Cloud](https://cloud.google.com/) as our cloud provider, under the -[Angular project](https://console.cloud.google.com/home/dashboard?project=internal-200822). -To access this project you need need to be logged in with a Google account that's a member of -team@angular.io. -For googlers this may be your google.com account, for others it is an angular.io account. - -In this project we have a number of Windows VMs running, each of them with several agents. -The `provision-windows-buildkite.ps1` file contains instructions on how to create new host VMs that -are fully configured to run the BuildKite agents as services. - -Our pipeline uses [docker-buildkite-plugin](https://github.com/buildkite-plugins/docker-buildkite-plugin) -to run build steps inside docker containers. -This way we achieve isolation and hermeticity. - -The `Dockerfile` file describes a custom Docker image that includes NodeJs, Yarn, and the Bazel -pre-requisites on Windows. - -To upload a new version of the docker image, follow any build instructions in `Dockerfile` and then -run `docker build -t angular/node-bazel-windows:NEW_VERSION`, followed by -`docker push angular/node-bazel-windows:NEW_VERSION`. -After being pushed it should be available online, and you can use the new version in `pipeline.yml`. - - -## Caretaker - -BuildKite status can be found at https://www.buildkitestatus.com/. - -Issues related to the BuildKite setup should be escalated to the Tools Team via the current -caretaker, followed by Alex Eagle and Filipe Silva. - -Support requests should be submitted via email to support@buildkite.com and cc Igor, Misko, Alex, -Jeremy and Manu - - -## Rollout strategy - -At the moment our BuildKite CI uses 1 host VM running 4 agents, thus being capable of 4 concurrent -builds. -The only test running is `bazel test //tools/ts-api-guardian:all`, and the PR check is not -mandatory. - -In the future we should add cache support to speed up the initial `yarn` install, and also Bazel -remote caching to speed up Bazel builds. - -After the current setup is verified as stable and reliable the GitHub PR check can become mandatory. - -The tests ran should also be expanded to cover most, if not all, of the Bazel tests. diff --git a/.buildkite/fix-msys64.cmd b/.buildkite/fix-msys64.cmd deleted file mode 100644 index 4722e6edd6..0000000000 --- a/.buildkite/fix-msys64.cmd +++ /dev/null @@ -1,6 +0,0 @@ -@echo off -REM Fix for https://github.com/docker/for-win/issues/2920 -REM echo "Fixing msys64 folder..." -REM Touch all .dll files inside C:\msys64\ -forfiles /p C:\msys64\ /s /m *.dll /c "cmd /c Copy /B @path+,, >NUL" -REM echo "Fixed msys64 folder." diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 58dd597bc4..0000000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,10 +0,0 @@ -steps: - - label: windows-test - commands: - - "yarn install --frozen-lockfile --non-interactive --network-timeout 100000" - - "yarn bazel test //tools/ts-api-guardian:all --noshow_progress" - plugins: - - docker#v2.1.0: - image: "filipesilva/node-bazel-windows:0.0.2" - agents: - windows: true diff --git a/.buildkite/provision-windows-buildkite.ps1 b/.buildkite/provision-windows-buildkite.ps1 deleted file mode 100644 index 7af03b2319..0000000000 --- a/.buildkite/provision-windows-buildkite.ps1 +++ /dev/null @@ -1,94 +0,0 @@ -# PowerShell script to provision a Windows Server with BuildKite -# This script follows https://buildkite.com/docs/agent/v3/windows. - -# Instructions - -# VM creation: -# In Google Cloud Platform, create a Compute Engine instance. -# We recommend machine type n1-standard-16 (16 vCPUs, 60 GB memory). -# Use a recent windows boot disk with container support such as -# "Windows Server version 1803 Datacenter Core for Containers", and add a 128GB SSD disk. -# Give it a name, then click "Create". - -# VM setup: -# In the Compute Engine menu, select "VM Instances". Click on the VM name you chose before. -# Click "Set Windows Password" to choose a username and password. -# Click RDP to open a remote desktop via browser, using the username and password. -# In the Windows command prompt start an elevated powershell by inputing -# "powershell -Command "Start-Process PowerShell -Verb RunAs" followed by Enter. -# Download and execute this script from GitHub, passing the token (mandatory), tags (optional) -# and number of agents (optional) as args: -# ``` -# Invoke-WebRequest -Uri https://raw.githubusercontent.com/angular/angular/master/.buildkite/provision-windows-buildkite.ps1 -OutFile provision.ps1 -# .\provision.ps1 -token "MY_TOKEN" -tags "windows=true,another_tag=true" -agents 4 -# ``` -# The VM should restart and be fully configured. - -# Creating extra VMs -# You can create an image of the current VM by following the instructions below. -# https://cloud.google.com/compute/docs/instances/windows/creating-windows-os-image -# Then create a new VM and choose "Custom images". - - -# Script proper. - -# Get the token and tags from arguments. -param ( - [Parameter(Mandatory=$true)][string]$token, - [string]$tags = "", - [Int]$agents = 1 -) - -# Allow HTTPS -[Net.ServicePointManager]::SecurityProtocol = "tls12, tls11, tls" - -# Helper to add to PATH. -# Will take current PATH so avoid running it after anything to modifies only the powershell session path. -function Add-Path ([string]$newPathItem) { - $Env:Path+= ";" + $newPathItem + ";" - [Environment]::SetEnvironmentVariable("Path",$env:Path, [System.EnvironmentVariableTarget]::Machine) -} - -# Install Git for Windows -Write-Host "Installing Git for Windows." -Invoke-WebRequest -Uri https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/Git-2.19.1-64-bit.exe -OutFile git.exe -.\git.exe /VERYSILENT /NORESTART /NOCANCEL /SP- /CLOSEAPPLICATIONS /RESTARTAPPLICATIONS /COMPONENTS="icons,ext\reg\shellhere,assoc,assoc_sh" /DIR="C:\git" -Add-Path "C:\git\bin" -# Sleep for 15s while git is installed. Trying to remove the git.exe before it finishes install causes an error. -Start-Sleep -s 15 -Remove-Item git.exe - -# Download NSSM (https://nssm.cc/) to run the BuildKite agent as a service. -Write-Host "Downloading NSSM." -Invoke-WebRequest -Uri https://nssm.cc/ci/nssm-2.24-101-g897c7ad.zip -OutFile nssm.zip -Expand-Archive -Path nssm.zip -DestinationPath C:\nssm -Add-Path "C:\nssm\nssm-2.24-101-g897c7ad\win64" -Remove-Item nssm.zip - -# Run the BuildKite agent install script -Write-Host "Installing BuildKite agent." -$env:buildkiteAgentToken = $token -$env:buildkiteAgentTags = $tags -Set-ExecutionPolicy Bypass -Scope Process -Force -iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/buildkite/agent/master/install.ps1')) - -# Configure the BuildKite agent clone and timestamp behavior -Add-Content C:\buildkite-agent\buildkite-agent.cfg "`ngit-clone-flags=--config core.autocrlf=input --config core.eol=lf --config core.longpaths=true --config core.symlinks=true`n" -Add-Content C:\buildkite-agent\buildkite-agent.cfg "`ntimestamp-lines=true`n" - -# Register the BuildKite agent service using NSSM, so that it persists through restarts and is -# restarted if the process dies. -for ($i=1; $i -le $agents; $i++) -{ - $agentName = "buildkite-agent-$i" - Write-Host "Registering $agentName as a service." - nssm.exe install $agentName "C:\buildkite-agent\bin\buildkite-agent.exe" "start" - nssm.exe set $agentName AppStdout "C:\buildkite-agent\$agentName.log" - nssm.exe set $agentName AppStderr "C:\buildkite-agent\$agentName.log" - nssm.exe status $agentName - nssm.exe start $agentName - nssm.exe status $agentName -} - -# Restart the machine. -Restart-Computer