build: fix build scripts on macOS (#33854)

In #33823, `scripts/package-builds.sh` (which is used by both
`build-packages-dist.sh` and `build-ivy-npm-packages.sh`) was updated to
use `realpath`. It turns out that `realpath` does not exist on macOS, so
the build scripts do not work there.

In order to fix this (and also reduce the likelihood of introducing
similar issues in the future), this commit changes these bash scripts to
Node.js scripts (using [ShellJS](https://github.com/shelljs/shelljs) for
a cross-platform implementation of Unix shell commands where necessary).

PR Close #33854

.circleci/config.yml

@@ -502,7 +502,7 @@ jobs:
       - setup_circleci_bazel_config
       - setup_bazel_rbe
 
-      - run: scripts/build-packages-dist.sh
+      - run: node scripts/build-packages-dist.js
 
       # Save the npm packages from //packages/... for other workflow jobs to read
       - persist_to_workspace:
@@ -530,7 +530,7 @@ jobs:
       - setup_circleci_bazel_config
       - setup_bazel_rbe
 
-      - run: scripts/build-ivy-npm-packages.sh
+      - run: node scripts/build-ivy-npm-packages.js
 
       # Save the npm packages from //packages/... for other workflow jobs to read
       - persist_to_workspace:

aio/tools/ng-packages-installer/index.js

@@ -15,7 +15,8 @@ const PACKAGE_JSON_REGEX = /^[^/]+\/package\.json$/;
 
 const ANGULAR_ROOT_DIR = path.resolve(__dirname, '../../..');
 const ANGULAR_DIST_PACKAGES = path.join(ANGULAR_ROOT_DIR, 'dist/packages-dist');
-const ANGULAR_DIST_PACKAGES_BUILD_CMD = path.join(ANGULAR_ROOT_DIR, 'scripts/build-packages-dist.sh');
+const ANGULAR_DIST_PACKAGES_BUILD_SCRIPT = path.join(ANGULAR_ROOT_DIR, 'scripts/build-packages-dist.js');
+const ANGULAR_DIST_PACKAGES_BUILD_CMD = `"${process.execPath}" "${ANGULAR_DIST_PACKAGES_BUILD_SCRIPT}"`;
 
 /**
  * A tool that can install Angular dependencies for a project from NPM or from the
@@ -173,13 +174,13 @@ class NgPackagesInstaller {
     const canBuild = process.platform !== 'win32';
 
     if (canBuild) {
-      this._log(`Building the Angular packages with: ${ANGULAR_DIST_PACKAGES_BUILD_CMD}`);
+      this._log(`Building the Angular packages with: ${ANGULAR_DIST_PACKAGES_BUILD_SCRIPT}`);
       shelljs.exec(ANGULAR_DIST_PACKAGES_BUILD_CMD);
     } else {
       this._warn([
         'Automatically building the local Angular packages is currently not supported on Windows.',
         `Please, ensure '${ANGULAR_DIST_PACKAGES}' exists and is up-to-date (e.g. by running ` +
-          `'${ANGULAR_DIST_PACKAGES_BUILD_CMD}' in Git Bash for Windows, Windows Subsystem for Linux or a Linux ` +
+          `'${ANGULAR_DIST_PACKAGES_BUILD_SCRIPT}' in Git Bash for Windows, Windows Subsystem for Linux or a Linux ` +
           'docker container or VM).',
         '',
         'Proceeding anyway...',

aio/tools/ng-packages-installer/index.spec.js

@@ -253,20 +253,21 @@ describe('NgPackagesInstaller', () => {
     };
 
     it('should build the local packages, when not on Windows', () => {
-      const buildScript = path.join(ngRootDir, 'scripts/build-packages-dist.sh');
+      const buildScript = path.join(ngRootDir, 'scripts/build-packages-dist.js');
+      const buildCmd = `"${process.execPath}" "${buildScript}"`;
 
       buildDistPackagesOnPlatform('linux');
-      expect(shelljs.exec).toHaveBeenCalledWith(buildScript);
+      expect(shelljs.exec).toHaveBeenCalledWith(buildCmd);
 
       shelljs.exec.calls.reset();
 
       buildDistPackagesOnPlatform('darwin');
-      expect(shelljs.exec).toHaveBeenCalledWith(buildScript);
+      expect(shelljs.exec).toHaveBeenCalledWith(buildCmd);
 
       shelljs.exec.calls.reset();
 
       buildDistPackagesOnPlatform('anythingButWindows :(');
-      expect(shelljs.exec).toHaveBeenCalledWith(buildScript);
+      expect(shelljs.exec).toHaveBeenCalledWith(buildCmd);
 
       // Ensure that the script does actually exist (e.g. it was not renamed/moved).
       fs.existsSync.and.callThrough();

docs/DEBUG_MATERIAL_IVY.md

@@ -4,7 +4,7 @@ Currently all changes to Ivy are validated against the test suite of the
 `angular/components` repository. In order to debug the `material-unit-tests` CI
 job, the following steps can be used:
 
-1\) Build the Ivy package output by running `./scripts/build-ivy-npm-packages.sh` in
+1\) Build the Ivy package output by running `node ./scripts/build-ivy-npm-packages.js` in
 the `angular/angular` repo.
 
 2\) Clone the `angular/components` repository if not done yet ([quick link to repo](https://github.com/angular/components)).
@@ -32,9 +32,9 @@ selecting a given test target.
 
 Here is an example of commands that run individual test targets. Note that it is
 **important** to specify the `--define=compile=aot` flag in order to run tests with Ivy.
-  
+
 ```bash
 yarn bazel test --define=compile=aot src/material/slider:unit_tests
 yarn bazel test --define=compile=aot src/cdk/a11y:unit_tests
 yarn bazel test --define=compile=aot src/material/toolbar:unit_tests
-``` 
+```

docs/DEVELOPER.md

@@ -69,7 +69,7 @@ yarn install
 To build Angular run:
 
 ```shell
-./scripts/build-packages-dist.sh
+node ./scripts/build-packages-dist.js
 ```
 
 * Results are put in the `dist/packages-dist` folder.
@@ -190,7 +190,7 @@ c. Some package managers (such as `pnpm` or `yarn pnp`) might not work correctly
 ### Publishing to GitHub repos
 You can also manually publish `*-builds` snapshots just like our CircleCI build does for upstream
 builds. Before being able to publish the packages, you need to build them locally by running the
-`./scripts/build-packages-dist.sh` script.
+`./scripts/build-packages-dist.js` script.
 
 First time, you need to create the GitHub repositories:
 

integration/cli-hello-world-ivy-compat/debug-test.sh

@@ -11,7 +11,7 @@ set -u -e -o pipefail
 
 cd "$(dirname "$0")"
 
-$(pwd)/../../scripts/build-packages-dist.sh
+node $(pwd)/../../scripts/build-packages-dist.js
 
 # Workaround https://github.com/yarnpkg/yarn/issues/2165
 # Yarn will cache file://dist URIs and not update Angular code
@@ -26,4 +26,4 @@ trap rm_cache EXIT
 rm -rf dist
 rm -rf node_modules
 yarn install --cache-folder $cache
-yarn test
+yarn test

integration/cli-hello-world-ivy-minimal/debug-test.sh

@@ -11,7 +11,7 @@ set -u -e -o pipefail
 
 cd "$(dirname "$0")"
 
-$(pwd)/../../scripts/build-packages-dist.sh
+node $(pwd)/../../scripts/build-packages-dist.js
 
 # Workaround https://github.com/yarnpkg/yarn/issues/2165
 # Yarn will cache file://dist URIs and not update Angular code
@@ -26,4 +26,4 @@ trap rm_cache EXIT
 rm -rf dist
 rm -rf node_modules
 yarn install --cache-folder $cache
-yarn test
+yarn test

integration/ivy-i18n/debug-test.sh

@@ -11,7 +11,7 @@ set -u -e -o pipefail
 
 cd "$(dirname "$0")"
 
-$(pwd)/../../scripts/build-packages-dist.sh
+node $(pwd)/../../scripts/build-packages-dist.js
 
 # Workaround https://github.com/yarnpkg/yarn/issues/2165
 # Yarn will cache file://dist URIs and not update Angular code
@@ -26,4 +26,4 @@ trap rm_cache EXIT
 rm -rf dist
 rm -rf node_modules
 yarn install --cache-folder $cache
-yarn test
+yarn test

integration/language_service_plugin/package.json

@@ -12,7 +12,7 @@
   },
   "scripts": {
     "build": "tsc -p tsconfig.json",
-    "build-dist": "../../scripts/build-packages-dist.sh && yarn install --check-files",
+    "build-dist": "node ../../scripts/build-packages-dist.js && yarn install --check-files",
     "cleanup": "rm -rf ti-*.log tsserver.log",
     "golden": "yarn build && node generate.js",
     "test": "yarn cleanup && yarn build && jasmine test.js"

integration/ngcc/debug-test.sh

@@ -11,7 +11,7 @@ set -u -e -o pipefail
 
 cd "$(dirname "$0")"
 
-$(pwd)/../../scripts/build-packages-dist.sh
+node $(pwd)/../../scripts/build-packages-dist.js
 
 # Workaround https://github.com/yarnpkg/yarn/issues/2165
 # Yarn will cache file://dist URIs and not update Angular code
@@ -26,4 +26,4 @@ trap rm_cache EXIT
 rm -rf dist
 rm -rf node_modules
 yarn install --cache-folder $cache
-yarn test
+yarn test

integration/run_tests.sh

@@ -24,12 +24,12 @@ if $CI; then
   # Determines the tests that need to be run for this shard index.
   TEST_DIRS=$(node ./get-sharded-tests.js --shardIndex ${SHARD_INDEX} --maxShards ${MAX_SHARDS})
 
-  # NB: we don't run build-packages-dist.sh because we expect that it was done
+  # NB: we don't run build-packages-dist.js because we expect that it was done
   # by an earlier job in the CircleCI workflow.
 else
   # Not on CircleCI so let's build the packages-dist directory.
   # This should be fast on incremental re-build.
-  ${basedir}/scripts/build-packages-dist.sh
+  node ${basedir}/scripts/build-packages-dist.js
 
   # If we aren't running on CircleCI, we do not shard tests because this would be the job of
   # Bazel eventually. For now, we just run all tests sequentially when running locally.

package.json

@@ -66,7 +66,7 @@
     "@types/node": "^12.11.1",
     "@types/selenium-webdriver": "3.0.7",
     "@types/semver": "^6.0.2",
-    "@types/shelljs": "^0.7.8",
+    "@types/shelljs": "^0.8.6",
     "@types/systemjs": "0.19.32",
     "@types/yargs": "^11.1.1",
     "@webcomponents/custom-elements": "^1.0.4",
@@ -110,7 +110,7 @@
     "rollup-plugin-sourcemaps": "^0.4.2",
     "rxjs": "^6.5.3",
     "selenium-webdriver": "3.5.0",
-    "shelljs": "^0.8.1",
+    "shelljs": "^0.8.3",
     "source-map": "^0.6.1",
     "source-map-support": "0.5.9",
     "systemjs": "0.18.10",

scripts/build-ivy-npm-packages.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+'use strict';
+
+const {buildTargetPackages} = require('./package-builder');
+
+
+// Build the ivy packages.
+buildTargetPackages('dist/packages-dist-ivy-aot', 'aot', 'Ivy AOT');

scripts/build-ivy-npm-packages.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-source $(dirname $0)/package-builder.sh
-
-# Build the ivy packages
-buildTargetPackages "dist/packages-dist-ivy-aot" "aot" "Ivy AOT"

scripts/build-packages-dist.js

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+'use strict';
+
+const {chmod, cp, mkdir, rm} = require('shelljs');
+const {
+  baseDir,
+  bazelBin,
+  bazelCmd,
+  buildTargetPackages,
+  exec,
+  scriptPath,
+} = require('./package-builder');
+
+
+// Build the legacy (view engine) npm packages into `dist/packages-dist/`.
+buildTargetPackages('dist/packages-dist', 'legacy', 'Production');
+
+// Build the `zone.js` npm package (into `dist/bin/packages/zone.js/npm_package/`), because it might
+// be needed by other scripts/tests.
+//
+// NOTE: The `zone.js` package is not built as part of `buildTargetPackages()` above, nor is it
+//       copied into the `dist/packages-dist/` directory (despite its source's being inside
+//       `packages/`), because it is not published to npm under the `@angular` scope (as happens for
+//       the rest of the packages).
+console.log('');
+console.log('##############################');
+console.log(`${scriptPath}:`);
+console.log('  Building zone.js npm package');
+console.log('##############################');
+exec(`${bazelCmd} build //packages/zone.js:npm_package`);
+
+// Copy artifacts to `dist/zone.js-dist/`, so they can be easier persisted on CI.
+const buildOutputDir = `${bazelBin}/packages/zone.js/npm_package`;
+const distTargetDir = `${baseDir}/dist/zone.js-dist/zone.js`;
+
+console.log(`# Copy artifacts to ${distTargetDir}`);
+mkdir('-p', distTargetDir);
+rm('-rf', distTargetDir);
+cp('-R', buildOutputDir, distTargetDir);
+chmod('-R', 'u+w', distTargetDir);

scripts/build-packages-dist.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-source $(dirname $0)/package-builder.sh
-
-# Build the legacy (view engine) npm packages into dist/packages-dist
-buildTargetPackages "dist/packages-dist" "legacy" "Production"
-
-# Build the `zone.js` npm package (into `dist/bin/packages/zone.js/npm_package/`), because it might be needed
-# by other scripts/tests.
-#
-# NOTE: The `zone.js` package is not built as part of `buildTargetPackages()` above, nor is it
-#       copied into the `dist/packages-dist/` directory (despite its source's being in `packages/`),
-#       because it is not published to npm under the `@angular` scope (as happens for the rest of
-#       the packages).
-echo ""
-echo "##############################"
-echo "${script_path}:"
-echo "  Building zone.js npm package"
-echo "##############################"
-yarn --silent bazel build //packages/zone.js:npm_package
-
-# Copy artifacts to `dist/zone.js-dist/`, so they can be easier persisted on CI.
-readonly buildOutputDir="$base_dir/dist/bin/packages/zone.js/npm_package"
-readonly distTargetDir="$base_dir/dist/zone.js-dist/zone.js"
-
-echo "# Copy artifacts to $distTargetDir"
-mkdir -p $distTargetDir
-rm -rf $distTargetDir
-cp -R $buildOutputDir $distTargetDir
-chmod -R u+w $distTargetDir

scripts/package-builder.js

@@ -0,0 +1,118 @@
+'use strict';
+
+// Build the dist/packages-dist directory in the same fashion as the legacy
+// /build.sh script, by building the npm packages with Bazel and copying files.
+// This is needed for scripts and tests which are not updated to the Bazel output
+// layout (which always matches the input layout).
+// Do not add new dependencies on this script, instead adapt scripts to use the
+// new layout, and write new tests as Bazel targets.
+//
+// Ideally integration tests should run under bazel, and just consume the npm
+// packages via `deps`. Until that works, we manually build the npm packages and then
+// copy the results to the appropriate `dist` location.
+
+const {execSync} = require('child_process');
+const {existsSync, statSync} = require('fs');
+const {resolve, relative} = require('path');
+const {chmod, cp, mkdir, rm, set} = require('shelljs');
+
+set('-e');
+
+
+/** @type {string} The absolute path to the project root directory. */
+const baseDir = resolve(`${__dirname}/..`);
+
+/** @type {string} The command to use for running bazel. */
+const bazelCmd = 'yarn --silent bazel';
+
+/** @type {string} The absolute path to the bazel-bin directory. */
+const bazelBin = exec(`${bazelCmd} info bazel-bin`, true);
+
+/**
+ * @type {string}
+ * The relative path to the entry script (i.e. the one loaded when the Node.js process launched).
+ * It is relative to `baseDir`.
+ */
+const scriptPath = relative(baseDir, require.main.filename);
+
+module.exports = {
+  baseDir,
+  bazelBin,
+  bazelCmd,
+  buildTargetPackages,
+  exec,
+  scriptPath,
+};
+
+/**
+ * Build the packages.
+ *
+ * @param {string} destPath Path to the output directory into which we copy the npm packages.
+ * @param {'legacy' | 'aot'} compileMode Either `legacy` (view engine) or `aot` (ivy).
+ * @param {string} description Human-readable description of the build.
+ */
+function buildTargetPackages(destPath, compileMode, description) {
+  console.log('##################################');
+  console.log(`${scriptPath}:`);
+  console.log('  Building @angular/* npm packages');
+  console.log(`  Mode: ${description}`);
+  console.log('##################################');
+
+  // List of targets to build, e.g. core, common, compiler, etc. Note that we want to also remove
+  // all carriage return (`\r`) characters form the query output, because otherwise the carriage
+  // return is part of the bazel target name and bazel will complain.
+  const getTargetsCmd = `${bazelCmd} query --output=label "attr('tags', '\\[.*release-with-framework.*\\]', //packages/...) intersect kind('.*_package', //packages/...)"`;
+  const targets = exec(getTargetsCmd, true).split(/\r?\n/);
+
+  // Use `--config=release` so that snapshot builds get published with embedded version info.
+  exec(`${bazelCmd} build --config=release --define=compile=${compileMode} ${targets.join(' ')}`);
+
+  // Create the output directory.
+  const absDestPath = `${baseDir}/${destPath}`;
+  if (!existsSync(absDestPath)) mkdir('-p', absDestPath);
+
+  targets.forEach(target => {
+    const pkg = target.replace(/\/\/packages\/(.*):npm_package/, '$1');
+
+    // Skip any that don't have an "npm_package" target.
+    const srcDir = `${bazelBin}/packages/${pkg}/npm_package`;
+    const destDir = `${absDestPath}/${pkg}`;
+
+    if (existsSync(srcDir) && statSync(srcDir).isDirectory()) {
+      console.log(`# Copy artifacts to ${destDir}`);
+      rm('-rf', destDir);
+      cp('-R', srcDir, destDir);
+      chmod('-R', 'u+w', destDir);
+    }
+  });
+}
+
+/**
+ * Execute a command synchronously.
+ *
+ * By default, the current process' stdout is used (and thus the output is not captured and returned
+ * to the caller). This is necessary for showing colors and modifying already printed output, for
+ * example to show progress.
+ *
+ * If the caller requests the output (via `captureStdout: true`), the command is run without
+ * printing anything to stdout and then (once the command has completed) the whole output is printed
+ * to stdout and returned to the caller.
+ *
+ * @param {string} cmd The command to run.
+ * @param {boolean} [captureStdout=false] Whether to return the output of the command.
+ * @return {string | undefined} The captured stdout output if `captureStdout: true` or `undefined`.
+ */
+function exec(cmd, captureStdout) {
+  const output = execSync(cmd, {
+    stdio: [
+      /* stdin  */ 'inherit',
+      /* stdout */ captureStdout ? 'pipe' : 'inherit',
+      /* stderr */ 'inherit',
+    ],
+  });
+
+  if (captureStdout) {
+    process.stdout.write(output);
+    return output.toString().trim();
+  }
+}

scripts/package-builder.sh

@@ -1,63 +0,0 @@
-#!/usr/bin/env bash
-
-# Build the dist/packages-dist directory in the same fashion as the legacy
-# /build.sh script, by building the npm packages with Bazel and copying files.
-# This is needed for scripts and tests which are not updated to the Bazel output
-# layout (which always matches the input layout).
-# Do not add new dependencies on this script, instead adapt scripts to use the
-# new layout, and write new tests as Bazel targets.
-#
-# Ideally integration tests should run under bazel, and just consume the npm
-# packages via `deps`. Until that works, we manually build the npm packages and then
-# copy the results to the appropriate `dist` location.
-
-set -u -e -o pipefail
-
-cd "$(dirname "$0")"
-
-# basedir is the workspace root
-readonly base_dir="$(dirname "$(pwd)")"
-readonly bazel_bin="$(yarn run -s bazel info bazel-bin)"
-readonly script_path="$0"
-
-function buildTargetPackages() {
-  # Path to the output directory into which we copy the npm packages.
-  dest_path="$1"
-
-  # Either "legacy" (view engine) or "aot" (ivy)
-  compile_mode="$2"
-
-  # Human-readable description of the build.
-  desc="$3"
-
-  echo "##################################"
-  echo "${script_path}:"
-  echo "  Building @angular/* npm packages"
-  echo "  Mode: ${desc}"
-  echo "##################################"
-
-  # List of targets to build, e.g. core, common, compiler, etc. Note that we want to
-  # remove all carriage return ("\r") characters form the query output because otherwise
-  # the carriage return is part of the bazel target name and bazel will complain.
-  targets=$(yarn run -s bazel query --output=label 'attr("tags", "\[.*release-with-framework.*\]", //packages/...) intersect kind(".*_package", //packages/...)' | tr -d "\r")
-
-  # Use --config=release so that snapshot builds get published with embedded version info
-  echo "$targets" | xargs yarn run -s bazel build --config=release --define=compile=${compile_mode}
-
-  [[ -d "${base_dir}/${dest_path}" ]] || mkdir -p ${base_dir}/${dest_path}
-
-  dirs=`echo "$targets" | sed -e 's/\/\/packages\/\(.*\):npm_package/\1/'`
-
-  for pkg in ${dirs}; do
-    # Skip any that don't have an "npm_package" target
-    src_dir="${bazel_bin}/packages/${pkg}/npm_package"
-    dest_dir="${base_dir}/${dest_path}/${pkg}"
-    if [[ -d ${src_dir} ]]; then
-      echo "# Copy artifacts to ${dest_dir}"
-      rm -rf ${dest_dir}
-      cp -R ${src_dir} ${dest_dir}
-      chmod -R u+w ${dest_dir}
-    fi
-  done
-}
-

yarn.lock

@@ -865,10 +865,10 @@
   resolved "https://registry.yarnpkg.com/@types/semver/-/semver-6.0.2.tgz#5e8b09f0e4af53034b1d0fb9977a277847836205"
   integrity sha512-G1Ggy7/9Nsa1Jt2yiBR2riEuyK2DFNnqow6R7cromXPMNynackRY1vqFTLz/gwnef1LHokbXThcPhqMRjUbkpQ==
 
-"@types/shelljs@^0.7.8":
+"@types/shelljs@^0.8.6":
-  version "0.7.9"
+  version "0.8.6"
-  resolved "https://registry.yarnpkg.com/@types/shelljs/-/shelljs-0.7.9.tgz#3abecb72d9cad9cd4b0e7cb86ed10a97d93ba602"
+  resolved "https://registry.yarnpkg.com/@types/shelljs/-/shelljs-0.8.6.tgz#45193a51df99e0f00513c39a2152832399783221"
-  integrity sha512-GwfXBWx+JgH+mrf35NnNFPFl6kQZgDQqZBUdWrHB1phulBbVpOwedZun7hZRyfTOxlicwo4ftsC1fpUZZIiN5w==
+  integrity sha512-svx2eQS268awlppL/P8wgDLBrsDXdKznABHJcuqXyWpSKJgE1s2clXlBvAwbO/lehTmG06NtEWJRkAk4tAgenA==
   dependencies:
     "@types/glob" "*"
     "@types/node" "*"
@@ -10322,10 +10322,10 @@ shell-quote@1.6.1:
     array-reduce "~0.0.0"
     jsonify "~0.0.0"
 
-shelljs@^0.8.1:
+shelljs@^0.8.3:
-  version "0.8.2"
+  version "0.8.3"
-  resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.2.tgz#345b7df7763f4c2340d584abb532c5f752ca9e35"
+  resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.3.tgz#a7f3319520ebf09ee81275b2368adb286659b097"
-  integrity sha512-pRXeNrCA2Wd9itwhvLp5LZQvPJ0wU6bcjaTMywHHGX5XWhVN2nzSu7WV0q+oUY7mGK3mgSkDDzP3MgjqdyIgbQ==
+  integrity sha512-fc0BKlAWiLpwZljmOvAOTE/gXawtCoNrP5oaY7KIaQbbyHeQVg01pSEuEGvGh3HEdBU4baCD7wQBwADmM/7f7A==
   dependencies:
     glob "^7.0.0"
     interpret "^1.0.0"