test(compiler-cli): create "full compile" compliance test rules (#39617)

This commit contains the basic runner logic and a couple of sample test cases
for the "full compile" compliance tests, where source files are compiled
to full definitions and checked against expectations.

PR Close #39617

.ng-dev/format.ts

@@ -18,6 +18,8 @@ export const format: FormatConfig = {
       '!**/*.d.ts',
       // Do not format generated ng-dev script
       '!dev-infra/ng-dev.js',
+      // Do not format compliance test-cases since they must match generated code
+      '!packages/compiler-cli/test/compliance/test_cases/**/*.js',
     ]
   },
   'buildifier': true

packages/compiler-cli/test/compliance/README.md

@@ -0,0 +1,157 @@
+# Compliance test-cases
+
+This directory contains rules, helpers and test-cases for the Angular compiler compliance tests.
+
+There are three different types of tests that are run based on file-based "test-cases".
+
+* **Full compile** - in this test the source files defined by the test-case are fully compiled by Angular.
+  The generated files are compared to "expected files" via a matching algorithm that is tolerant to
+  whitespace and variable name changes.
+* **Partial compile** - in this test the source files defined by the test-case are "partially" compiled by
+  Angular to produce files that can be published. These partially compiled files are compared directly
+  against "golden files" to ensure that we do not inadvertently break the public API of partial
+  declarations.
+* **Linked** - in this test the golden files mentioned in the previous bullet point, are passed to the
+  Angular linker, which generates files that are comparable to the fully compiled files. These linked
+  files are compared against the "expected files" in the same way as in the "full compile" tests.
+
+This way the compliance tests are able to check each mode and stage of compilation is accurate and does
+not change unexpectedly.
+
+
+## Defining a test-case
+
+To define a test-case, create a new directory below `test_cases`. In this directory
+
+* add a new file called `TEST_CASES.json`. The format of this file is described below.
+* add an empty `GOLDEN_PARTIAL.js` file. This file will be updated by the tooling later.
+* add any `inputFiles` that will be compiled as part of the test-case.
+* add any `expected` files that will be compared to the files generated by compiling the source files.
+
+
+### TEST_CASES.json format
+
+The `TEST_CASES.json` defines an object with one or more test-case definitions in the `cases` property.
+
+Each test-case can specify:
+
+* A `description` of the test.
+* The `inputFiles` that will be compiled.
+* Additional `compilerOptions` and `angularCompilerOptions` that are passed to the compiler.
+* A collection of `expectations` definitions that will be checked against the generated files.
+
+Note that there is a JSON schema for the `TEST_CASES.json` file stored at `test_cases/test_case_schema.json`.
+You should add a link to this schema at the top of your `TEST_CASES.json` file to provide
+validation and intellisense for this file in your IDE.
+
+For example:
+
+```json
+{
+  "$schema": "../test_case_schema.json",
+  "cases": [
+    {
+      "description": "description of the test - equivalent to an `it` clause message.",
+      "inputFiles": ["abc.ts"],
+      "expectations": [
+        {
+          "failureMessage": "message to display if this expectation fails",
+          "files": [
+            { "expected": "xyz.js", "generated": "abc.js" }, ...
+          ]
+        }, ...
+      ],
+      "compilerOptions": { ... },
+      "angularCompilerOptions": { ... }
+    }
+  ]
+}
+```
+
+### Input files
+
+The input files are the source file that will be compiled as part of this test-case.
+Input files should be stored in the directory next to the `TEST_CASES.json`.
+The paths to the input files should be listed in the `inputFiles` property of `TEST_CASES.json`.
+The paths are relative to the `TEST_CASES.json` file.
+
+If no `inputFiles` property is provided, the default is `["test.ts"]`.
+
+
+### Expectations
+
+An expectation consists of a collection of expected `files` pairs, and a `failureMessage`, which
+is displayed if the expectation check fails.
+
+Each file-pair consists of a path to a `generated` file (relative to the build output folder),
+and a path to an `expected` file (relative to the test case).
+
+The `generated` file is checked to see if it "matches" the `expected` file. The matching is
+resilient to whitespace and variable name changes.
+
+If no `failureMessage` property is provided, the default is `"Incorrect generated output."`.
+
+If no `files` property is provided, the default is a a collection of objects `{expected, generated}`,
+where `expected` and `generated` are computed by taking each path in the `inputFiles` collection
+and replacing the `.ts` extension with `.js`.
+
+
+## Running tests
+
+The simplest way to run all the compliance tests is:
+
+```sh
+yarn test-ivy-aot //packages/compiler-cli/test/compliance/...
+```
+
+If you only want to run one of the three types of test you can be more specific:
+
+```sh
+yarn test-ivy-aot //packages/compiler-cli/test/compliance/full
+yarn test-ivy-aot //packages/compiler-cli/test/compliance/linked
+yarn test-ivy-aot //packages/compiler-cli/test/compliance/test_cases/...
+```
+
+(The last command runs the partial compilation tests.)
+
+
+## Updating a golden partial file
+
+There is one golden partial file per `TEST_CASES.json` file. So even if this file defines multiple
+test-cases, which each contain multiple input files, there will only be one golden file.
+
+The golden file is generated by the tooling and should not be modified manually.
+
+When you first create a test-case, with an empty `PARTIAL_GOLDEN.js` file, or a change is made to
+the generated partial output, we must update the `PARTIAL_GOLDEN.js` file.
+
+This is done by running a specific bazel rule of the form:
+
+```sh
+bazel run //packages/compiler-cli/test/compliance/test_cases:<path/to/test_case>.golden.update
+```
+
+where to replace `<path/to/test_case>` with the path (relative to `test_cases`) of the directory
+that contains the `PARTIAL_GOLDEN.js` to update.
+
+
+## Debugging test-cases
+
+Compliance tests are basically `jasmine_node_test` rules. As such, they can be debugged
+just like any other `jasmine_node_test`.  The standard approach is to add `--config=debug`
+to the Bazel test command.
+
+It is useful when debugging to focus on a single test-case.
+
+
+### Focusing test-cases
+
+You can focus a test case by setting `"focusTest": true` in the `TEST_CASES.json` file.
+This is equivalent to using jasmine `fit()`.
+
+
+### Excluding test-cases
+
+You can exclude a test case by setting `"excludeTest": true` in the `TEST_CASES.json` file.
+This is equivalent to using jasmine `xit()`.
+

packages/compiler-cli/test/compliance/full/BUILD.bazel

@@ -0,0 +1,27 @@
+load("//tools:defaults.bzl", "jasmine_node_test", "ts_library")
+
+ts_library(
+    name = "test_lib",
+    testonly = True,
+    srcs = ["full_compile_spec.ts"],
+    deps = [
+        "//packages/compiler-cli/src/ngtsc/file_system",
+        "//packages/compiler-cli/test/compliance/test_helpers",
+    ],
+)
+
+jasmine_node_test(
+    name = "full",
+    bootstrap = ["//tools/testing:node_no_angular_es5"],
+    data = [
+        "//packages/compiler-cli/test/compliance/test_cases",
+        "//packages/compiler-cli/test/ngtsc/fake_core:npm_package",
+    ],
+    shard_count = 2,
+    tags = [
+        "ivy-only",
+    ],
+    deps = [
+        ":test_lib",
+    ],
+)

packages/compiler-cli/test/compliance/full/full_compile_spec.ts

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {FileSystem} from '../../../src/ngtsc/file_system';
+import {compileTest} from '../test_helpers/compile_test';
+import {ComplianceTest} from '../test_helpers/get_compliance_tests';
+import {runTests} from '../test_helpers/test_runner';
+
+runTests('full compile', compileTests);
+
+/**
+ * Fully compile all the input files in the given `test`.
+ *
+ * @param fs The mock file-system where the input files can be found.
+ * @param test The compliance test whose input files should be compiled.
+ */
+function compileTests(fs: FileSystem, test: ComplianceTest): void {
+  compileTest(fs, test.inputFiles, test.compilerOptions, test.angularCompilerOptions);
+}

packages/compiler-cli/test/compliance/test_cases/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin")
+
+package(default_visibility = ["//packages/compiler-cli/test/compliance:__subpackages__"])
+
+# Normally we would use `file_group` here but this doesn't work on Windows because there
+# appears to be a bug with making the `runfiles` folder available.
+# See https://github.com/bazelbuild/rules_nodejs/issues/1689
+copy_to_bin(
+    name = "test_cases",
+    srcs = glob(["*/**"]),
+)

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/GOLDEN_PARTIAL.js

@@ -0,0 +1,47 @@
+/****************************************************************************************************
+ * PARTIAL FILE: test.js
+ ****************************************************************************************************/
+import { Component, NgModule } from '@angular/core';
+import * as i0 from "@angular/core";
+export class MyApp {
+    constructor() {
+        this.list = [];
+    }
+}
+MyApp.ɵfac = function MyApp_Factory(t) { return new (t || MyApp)(); };
+MyApp.ɵcmp = i0.ɵɵdefineComponent({ type: MyApp, selectors: [["my-app"]], decls: 1, vars: 9, template: function MyApp_Template(rf, ctx) { if (rf & 1) {
+        i0.ɵɵtext(0);
+    } if (rf & 2) {
+        i0.ɵɵtextInterpolateV([" ", ctx.list[0], " ", ctx.list[1], " ", ctx.list[2], " ", ctx.list[3], " ", ctx.list[4], " ", ctx.list[5], " ", ctx.list[6], " ", ctx.list[7], " ", ctx.list[8], " "]);
+    } }, encapsulation: 2 });
+/*@__PURE__*/ (function () { i0.ɵsetClassMetadata(MyApp, [{
+        type: Component,
+        args: [{
+                selector: 'my-app',
+                template: ' {{list[0]}} {{list[1]}} {{list[2]}} {{list[3]}} {{list[4]}} {{list[5]}} {{list[6]}} {{list[7]}} {{list[8]}} '
+            }]
+    }], null, null); })();
+export class MyModule {
+}
+MyModule.ɵmod = i0.ɵɵdefineNgModule({ type: MyModule });
+MyModule.ɵinj = i0.ɵɵdefineInjector({ factory: function MyModule_Factory(t) { return new (t || MyModule)(); } });
+(function () { (typeof ngJitMode === "undefined" || ngJitMode) && i0.ɵɵsetNgModuleScope(MyModule, { declarations: [MyApp] }); })();
+/*@__PURE__*/ (function () { i0.ɵsetClassMetadata(MyModule, [{
+        type: NgModule,
+        args: [{ declarations: [MyApp] }]
+    }], null, null); })();
+
+/****************************************************************************************************
+ * PARTIAL FILE: test.d.ts
+ ****************************************************************************************************/
+import * as i0 from "@angular/core";
+export declare class MyApp {
+    list: any[];
+    static ɵfac: i0.ɵɵFactoryDef<MyApp, never>;
+    static ɵcmp: i0.ɵɵComponentDefWithMeta<MyApp, "my-app", never, {}, {}, never, never>;
+}
+export declare class MyModule {
+    static ɵmod: i0.ɵɵNgModuleDefWithMeta<MyModule, [typeof MyApp], never, never>;
+    static ɵinj: i0.ɵɵInjectorDef<MyModule>;
+}
+

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/TEST_CASES.json

@@ -0,0 +1,13 @@
+{
+  "$schema": "../test_case_schema.json",
+  "cases": [
+    {
+      "description": "should generate a correct call to `ɵɵtextInterpolateV()` with more than 8 interpolations",
+      "expectations": [
+        {
+          "failureMessage": "Incorrect `ɵɵtextInterpolateV()` call"
+        }
+      ]
+    }
+  ]
+}

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/test.js

@@ -0,0 +1,11 @@
+function MyApp_Template(rf, ctx) {
+  if (rf & 1) {
+    $i0$.ɵɵtext(0);
+  }
+  if (rf & 2) {
+    $i0$.ɵɵtextInterpolateV([
+      " ", ctx.list[0], " ", ctx.list[1], " ", ctx.list[2], " ", ctx.list[3], " ", ctx.list[4], " ",
+      ctx.list[5], " ", ctx.list[6], " ", ctx.list[7], " ", ctx.list[8], " "
+    ]);
+  }
+}

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/test.ts

@@ -0,0 +1,14 @@
+import {Component, NgModule} from '@angular/core';
+
+@Component({
+  selector: 'my-app',
+  template:
+      ' {{list[0]}} {{list[1]}} {{list[2]}} {{list[3]}} {{list[4]}} {{list[5]}} {{list[6]}} {{list[7]}} {{list[8]}} '
+})
+export class MyApp {
+  list: any[] = [];
+}
+
+@NgModule({declarations: [MyApp]})
+export class MyModule {
+}

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/GOLDEN_PARTIAL.js

@@ -0,0 +1,50 @@
+/****************************************************************************************************
+ * PARTIAL FILE: test.js
+ ****************************************************************************************************/
+import { Component, Directive, NgModule } from '@angular/core';
+import * as i0 from "@angular/core";
+export class I18nDirective {
+}
+I18nDirective.ɵfac = function I18nDirective_Factory(t) { return new (t || I18nDirective)(); };
+I18nDirective.ɵdir = i0.ɵɵngDeclareDirective({ version: 1, type: I18nDirective, selector: "[i18n]", ngImport: i0 });
+/*@__PURE__*/ (function () { i0.ɵsetClassMetadata(I18nDirective, [{
+        type: Directive,
+        args: [{ selector: '[i18n]' }]
+    }], null, null); })();
+export class MyComponent {
+}
+MyComponent.ɵfac = function MyComponent_Factory(t) { return new (t || MyComponent)(); };
+MyComponent.ɵcmp = i0.ɵɵdefineComponent({ type: MyComponent, selectors: [["my-component"]], decls: 1, vars: 0, template: function MyComponent_Template(rf, ctx) { if (rf & 1) {
+        i0.ɵɵelement(0, "div");
+    } }, encapsulation: 2 });
+/*@__PURE__*/ (function () { i0.ɵsetClassMetadata(MyComponent, [{
+        type: Component,
+        args: [{ selector: 'my-component', template: '<div i18n></div>' }]
+    }], null, null); })();
+export class MyModule {
+}
+MyModule.ɵmod = i0.ɵɵdefineNgModule({ type: MyModule });
+MyModule.ɵinj = i0.ɵɵdefineInjector({ factory: function MyModule_Factory(t) { return new (t || MyModule)(); } });
+(function () { (typeof ngJitMode === "undefined" || ngJitMode) && i0.ɵɵsetNgModuleScope(MyModule, { declarations: [I18nDirective, MyComponent] }); })();
+/*@__PURE__*/ (function () { i0.ɵsetClassMetadata(MyModule, [{
+        type: NgModule,
+        args: [{ declarations: [I18nDirective, MyComponent] }]
+    }], null, null); })();
+
+/****************************************************************************************************
+ * PARTIAL FILE: test.d.ts
+ ****************************************************************************************************/
+import * as i0 from "@angular/core";
+export declare class I18nDirective {
+    static ɵfac: i0.ɵɵFactoryDef<I18nDirective, never>;
+    static ɵdir: i0.ɵɵDirectiveDefWithMeta<I18nDirective, "[i18n]", never, {}, {}, never>;
+}
+export declare class MyComponent {
+    static ɵfac: i0.ɵɵFactoryDef<MyComponent, never>;
+    static ɵcmp: i0.ɵɵComponentDefWithMeta<MyComponent, "my-component", never, {}, {}, never, never>;
+}
+export declare class MyModule {
+    static ɵmod: i0.ɵɵNgModuleDefWithMeta<MyModule, [typeof I18nDirective, typeof MyComponent], never, never>;
+    static ɵinj: i0.ɵɵInjectorDef<MyModule>;
+}
+

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/TEST_CASES.json

@@ -0,0 +1,28 @@
+{
+  "$schema": "../../test_case_schema.json",
+  "cases": [
+    {
+      "description": "should not match directives on i18n attribute",
+      "expectations": [
+        {
+          "failureMessage": "Incorrect ChildComponent.ɵcmp",
+          "files": [
+            {
+              "expected": "component.js",
+              "generated": "test.js"
+            }
+          ]
+        },
+        {
+          "failureMessage": "Incorrect ChildComponent.ɵfac",
+          "files": [
+            {
+              "expected": "factory.js",
+              "generated": "test.js"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/component.js

@@ -0,0 +1,12 @@
+MyComponent.ɵcmp = $r3$.ɵɵdefineComponent({
+  type: MyComponent,
+  selectors: [["my-component"]],
+  decls: 1,
+  vars: 0,
+  template: function MyComponent_Template(rf, ctx) {
+    if (rf & 1) {
+      $r3$.ɵɵelement(0, "div");
+    }
+  },
+  encapsulation: 2
+});

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/factory.js

@@ -0,0 +1,3 @@
+MyComponent.ɵfac = function MyComponent_Factory(t) {
+  return new (t || MyComponent)();
+};

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/test.ts

@@ -0,0 +1,13 @@
+import {Component, Directive, NgModule} from '@angular/core';
+
+@Directive({selector: '[i18n]'})
+export class I18nDirective {
+}
+
+@Component({selector: 'my-component', template: '<div i18n></div>'})
+export class MyComponent {
+}
+
+@NgModule({declarations: [I18nDirective, MyComponent]})
+export class MyModule {
+}

packages/compiler-cli/test/compliance/test_cases/test_case_schema.json

@@ -0,0 +1,89 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://angular.io/internal/compliance-test-cases.json",
+  "title": "Compliance test cases",
+  "description": "Describes configuration for an Angular compliance test",
+  "type": "object",
+  "properties": {
+    "cases": {
+      "title": "A collection of test cases",
+      "type": "array",
+      "items": {
+        "title": "A test case definition",
+        "type": "object",
+        "required": [
+          "description"
+        ],
+        "properties": {
+          "description": {
+            "title": "A description of the test case",
+            "description": "This will be used as the message in an `it()` clause.",
+            "type": "string"
+          },
+          "inputFiles": {
+            "title": "A collection of source files to compile",
+            "type": "array",
+            "items": {
+              "title": "Path (relative to the test case) of a source file to compile",
+              "type": "string"
+            }
+          },
+          "expectations": {
+            "title": "A collection of expectations for this test case",
+            "type": "array",
+            "items": {
+              "title": "An expectation to check for this test case",
+              "type": "object",
+              "properties": {
+                "failureMessage": {
+                  "title": "The message to display if this expectation fails",
+                  "type": "string",
+                  "default": "Incorrect generated output."
+                },
+                "files": {
+                  "title": "A collection of expected-generated file path pairs",
+                  "type": "array",
+                  "items": {
+                    "title": "A pair of expected-generated file paths to be compared",
+                    "type": "object",
+                    "required": [
+                      "expected",
+                      "generated"
+                    ],
+                    "properties": {
+                      "expected": {
+                        "title": "A path (relative to the test case) where an file containing expected output can be found",
+                        "type": "string"
+                      },
+                      "generated": {
+                        "title": "A path (relative to the build output directory) where the compiled file can be found.",
+                        "type": "string"
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "compilerOptions": {
+            "title": "Additional options to pass to the TypeScript compiler",
+            "type": "object"
+          },
+          "angularCompilerOptions": {
+            "title": "Additional options to pass to the Angular compiler",
+            "type": "object"
+          },
+          "focusTest": {
+            "title": "Set to true to focus on this test - equivalent to jasmine's `fit()` function",
+            "type": "boolean"
+          },
+          "excludeTest": {
+            "title": "Set to true to exclude this test - equivalent to jasmine's `xit()` function",
+            "type": "boolean"
+          }
+        }
+      },
+      "minItems": 1
+    }
+  }
+}

packages/compiler-cli/test/compliance/test_helpers/BUILD.bazel

@@ -0,0 +1,21 @@
+load("//tools:defaults.bzl", "ts_library")
+
+ts_library(
+    name = "test_helpers",
+    testonly = True,
+    srcs = glob(
+        ["**/*.ts"],
+    ),
+    visibility = [
+        "//packages/compiler-cli/test/compliance:__subpackages__",
+    ],
+    deps = [
+        "//packages:types",
+        "//packages/compiler",
+        "//packages/compiler-cli",
+        "//packages/compiler-cli/src/ngtsc/file_system",
+        "//packages/compiler-cli/src/ngtsc/file_system/testing",
+        "//packages/compiler-cli/test/helpers",
+        "@npm//typescript",
+    ],
+)

packages/compiler-cli/test/compliance/test_helpers/check_expectations.ts

@@ -0,0 +1,45 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {FileSystem} from '../../../src/ngtsc/file_system';
+import {getBuildOutputDirectory} from './compile_test';
+import {expectEmit} from './expect_emit';
+import {ExpectedFile} from './get_compliance_tests';
+
+/**
+ * Check that each of the generated files matches the expected files.
+ *
+ * @param fs The mock file-system that holds the expected and generated files to compare.
+ * @param testPath Path to the current test case (relative to the basePath).
+ * @param failureMessage The message to display if the expectation fails.
+ * @param expectedFiles The list of expected-generated pairs to compare.
+ */
+export function checkExpectations(
+    fs: FileSystem, testPath: string, failureMessage: string, expectedFiles: ExpectedFile[]): void {
+  const builtDirectory = getBuildOutputDirectory(fs);
+  for (const expectedFile of expectedFiles) {
+    const expectedPath = fs.resolve(expectedFile.expected);
+    if (!fs.exists(expectedPath)) {
+      throw new Error(`The expected file at ${
+          expectedPath} does not exist. Please check the TEST_CASES.json file for this test case.`);
+    }
+
+    const generatedPath = fs.resolve(builtDirectory, expectedFile.generated);
+    if (!fs.exists(generatedPath)) {
+      throw new Error(`The generated file at ${
+          generatedPath} does not exist. Perhaps there is no matching input source file in the TEST_CASES.json file for this test case.`);
+    }
+
+    const expected = fs.readFile(expectedPath);
+    const generated = fs.readFile(generatedPath);
+
+    expectEmit(
+        generated, expected,
+        `When checking against expected file "${testPath}/${expectedFile.expected}"\n` +
+            failureMessage);
+  }
+}

packages/compiler-cli/test/compliance/test_helpers/compile_test.ts

@@ -0,0 +1,112 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import * as ts from 'typescript';
+
+import {AbsoluteFsPath, FileSystem, NgtscCompilerHost} from '../../../src/ngtsc/file_system';
+import {initMockFileSystem} from '../../../src/ngtsc/file_system/testing';
+import {performCompilation} from '../../../src/perform_compile';
+import {CompilerOptions} from '../../../src/transformers/api';
+import {loadStandardTestFiles, loadTestDirectory} from '../../helpers';
+
+import {ConfigOptions} from './get_compliance_tests';
+
+/**
+ * Setup a mock file-system that is used to generate the partial files.
+ *
+ * @param realTestPath Absolute path (on the real file-system) to the test case being processed.
+ * @returns a mock file-system containing the test case files.
+ */
+export function initMockTestFileSystem(realTestPath: AbsoluteFsPath): FileSystem {
+  const fs = initMockFileSystem('Native');
+  const testFiles = loadStandardTestFiles();
+  fs.init(testFiles);
+  loadTestDirectory(fs, realTestPath, getRootDirectory(fs));
+  return fs;
+}
+
+/**
+ * Compile the input source `files` stored in `fs`, writing the generated files to `fs`.
+ *
+ * @param fs The mock file-system where the input and generated files live.
+ * @param files An array of paths (relative to the testPath) of input files to be compiled.
+ * @param compilerOptions Any extra options to pass to the TypeScript compiler.
+ * @param angularCompilerOptions Any extra options to pass to the Angular compiler.
+ * @returns A collection of paths of the generated files (absolute within the mock file-system).
+ */
+export function compileTest(
+    fs: FileSystem, files: string[], compilerOptions: ConfigOptions|undefined,
+    angularCompilerOptions: ConfigOptions|undefined): AbsoluteFsPath[] {
+  const rootDir = getRootDirectory(fs);
+  const outDir = getBuildOutputDirectory(fs);
+  const options = getOptions(rootDir, outDir, compilerOptions, angularCompilerOptions);
+  const rootNames = files.map(f => fs.resolve(f));
+  const host = new NgtscCompilerHost(fs, options);
+  const {diagnostics, emitResult} = performCompilation({rootNames, host, options});
+  if (diagnostics.length > 0) {
+    console.warn(diagnostics.map(d => d.messageText).join('\n'));
+  }
+  return emitResult!.emittedFiles!.map(p => fs.resolve(rootDir, p));
+}
+
+/**
+ * Gets an absolute path (in the mock file-system) of the root directory where the compilation is to
+ * be done.
+ *
+ * @param fs the mock file-system where the compilation is happening.
+ */
+export function getRootDirectory(fs: FileSystem): AbsoluteFsPath {
+  return fs.resolve('/');
+}
+
+/**
+ * Gets an absolute path (in the mock file-system) of the directory where the compiled files are
+ * stored.
+ *
+ * @param fs the mock file-system where the compilation is happening.
+ */
+export function getBuildOutputDirectory(fs: FileSystem): AbsoluteFsPath {
+  return fs.resolve('/built');
+}
+
+/**
+ * Get the options object to pass to the compiler.
+ *
+ * @param rootDir The absolute path (within the mock file-system) that is the root of the
+ *     compilation.
+ * @param outDir The absolute path (within the mock file-system) where compiled files will be
+ *     written.
+ * @param compilerOptions Additional options for the TypeScript compiler.
+ * @param angularCompilerOptions Additional options for the Angular compiler.
+ */
+function getOptions(
+    rootDir: AbsoluteFsPath, outDir: AbsoluteFsPath, compilerOptions: ConfigOptions|undefined,
+    angularCompilerOptions: ConfigOptions|undefined): CompilerOptions {
+  return {
+    emitDecoratorMetadata: true,
+    experimentalDecorators: true,
+    skipLibCheck: true,
+    noImplicitAny: true,
+    noEmitOnError: true,
+    listEmittedFiles: true,
+    strictNullChecks: true,
+    outDir,
+    rootDir,
+    baseUrl: '.',
+    allowJs: true,
+    declaration: true,
+    target: ts.ScriptTarget.ES2015,
+    newLine: ts.NewLineKind.LineFeed,
+    module: ts.ModuleKind.ES2015,
+    moduleResolution: ts.ModuleResolutionKind.NodeJs,
+    typeRoots: ['node_modules/@types'],
+    ...ts.convertCompilerOptionsFromJson({compilerOptions}, rootDir).options,
+    enableIvy: true,
+    ivyTemplateTypeCheck: false,
+    ...angularCompilerOptions,
+  };
+}

packages/compiler-cli/test/compliance/test_helpers/expect_emit.ts

@@ -0,0 +1,197 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {escapeRegExp} from '@angular/compiler/src/util';
+
+const IDENTIFIER = /[A-Za-z_$ɵ][A-Za-z0-9_$]*/;
+const OPERATOR =
+    /!|\?|%|\*|\/|\^|&&?|\|\|?|\(|\)|\{|\}|\[|\]|:|;|<=?|>=?|={1,3}|!==?|=>|\+\+?|--?|@|,|\.|\.\.\.|`|\\'/;
+const STRING = /'(\\'|[^'])*'|"(\\"|[^"])*"/;
+const BACKTICK_STRING = /\\`(([\s\S]*?)(\$\{[^}]*?\})?)*?[^\\]\\`/;
+const BACKTICK_INTERPOLATION = /(\$\{[^}]*\})/;
+const NUMBER = /\d+/;
+
+const ELLIPSIS = '…';
+const TOKEN = new RegExp(
+    `\\s*((${IDENTIFIER.source})|(${BACKTICK_STRING.source})|(${OPERATOR.source})|(${
+        STRING.source})|${NUMBER.source}|${ELLIPSIS})\\s*`,
+    'y');
+
+type Piece = string|RegExp;
+
+const SKIP = /(?:.|\n|\r)*/;
+
+const ERROR_CONTEXT_WIDTH = 30;
+// Transform the expected output to set of tokens
+function tokenize(text: string): Piece[] {
+  // TOKEN.lastIndex is stateful so we cache the `lastIndex` and restore it at the end of the call.
+  const lastIndex = TOKEN.lastIndex;
+  TOKEN.lastIndex = 0;
+
+  let match: RegExpMatchArray|null;
+  let tokenizedTextEnd = 0;
+  const pieces: Piece[] = [];
+
+  while ((match = TOKEN.exec(text)) !== null) {
+    const [fullMatch, token] = match;
+    if (token === 'IDENT') {
+      pieces.push(IDENTIFIER);
+    } else if (token === ELLIPSIS) {
+      pieces.push(SKIP);
+    } else if (match = BACKTICK_STRING.exec(token)) {
+      pieces.push(...tokenizeBackTickString(token));
+    } else {
+      pieces.push(token);
+    }
+    tokenizedTextEnd += fullMatch.length;
+  }
+
+  if (pieces.length === 0 || tokenizedTextEnd < text.length) {
+    // The new token that could not be found is located after the
+    // last tokenized character.
+    const from = tokenizedTextEnd;
+    const to = from + ERROR_CONTEXT_WIDTH;
+    throw Error(
+        `Invalid test, no token found for "${text[tokenizedTextEnd]}" ` +
+        `(context = '${text.substr(from, to)}...'`);
+  }
+  // Reset the lastIndex in case we are in a recursive `tokenize()` call.
+  TOKEN.lastIndex = lastIndex;
+
+  return pieces;
+}
+
+/**
+ * Back-ticks are escaped as "\`" so we must strip the backslashes.
+ * Also the string will likely contain interpolations and if an interpolation holds an
+ * identifier we will need to match that later. So tokenize the interpolation too!
+ */
+function tokenizeBackTickString(str: string): Piece[] {
+  const pieces: Piece[] = ['`'];
+  // Unescape backticks that are inside the backtick string
+  // (we had to double escape them in the test string so they didn't look like string markers)
+  str = str.replace(/\\\\\\`/g, '\\`');
+  const backTickPieces = str.slice(2, -2).split(BACKTICK_INTERPOLATION);
+  backTickPieces.forEach((backTickPiece) => {
+    if (BACKTICK_INTERPOLATION.test(backTickPiece)) {
+      // An interpolation so tokenize this expression
+      pieces.push(...tokenize(backTickPiece));
+    } else {
+      // Not an interpolation so just add it as a piece
+      pieces.push(backTickPiece);
+    }
+  });
+  pieces.push('`');
+  return pieces;
+}
+
+export function expectEmit(
+    source: string, expected: string, description: string,
+    assertIdentifiers?: {[name: string]: RegExp}) {
+  expected = expected
+                 // turns `// ...` into `…`
+                 .replace(/\/\/\s*\.\.\./g, ELLIPSIS)
+                 // remove `// TODO` comment lines
+                 .replace(/\/\/\s*TODO.*?\n/g, '')
+                 // remove `// NOTE` comment lines
+                 .replace(/\/\/\s*NOTE.*?\n/g, '');
+
+  const pieces = tokenize(expected);
+  const {regexp, groups} = buildMatcher(pieces);
+  const matches = source.match(regexp);
+  if (matches === null) {
+    let last: number = 0;
+    for (let i = 1; i < pieces.length; i++) {
+      const {regexp} = buildMatcher(pieces.slice(0, i));
+      const m = source.match(regexp);
+      const expectedPiece = pieces[i - 1] == IDENTIFIER ? '<IDENT>' : pieces[i - 1];
+      if (!m) {
+        // display at most `contextLength` characters of the line preceding the error location
+        const contextLength = 50;
+        const fullContext = source.substring(source.lastIndexOf('\n', last) + 1, last);
+        const context = fullContext.length > contextLength ?
+            `...${fullContext.substr(-contextLength)}` :
+            fullContext;
+        throw new Error(
+            `${description}:\nFailed to find "${expectedPiece}" after "${context}" in:\n'${
+                source.substr(
+                    0, last)}[<---HERE expected "${expectedPiece}"]${source.substr(last)}'`);
+        return;
+      } else {
+        last = (m.index || 0) + m[0].length;
+      }
+    }
+    throw new Error(
+        `Test helper failure: Expected expression failed but the reporting logic could not find where it failed in: ${
+            source}`);
+  } else {
+    if (assertIdentifiers) {
+      // It might be possible to add the constraints in the original regexp (see `buildMatcher`)
+      // by transforming the assertion regexps when using anchoring, grouping, back references,
+      // flags, ...
+      //
+      // Checking identifiers after they have matched allows for a simple and flexible
+      // implementation.
+      // The overall performance are not impacted when `assertIdentifiers` is empty.
+      const ids = Object.keys(assertIdentifiers);
+      for (let i = 0; i < ids.length; i++) {
+        const id = ids[i];
+        if (groups.has(id)) {
+          const name = matches[groups.get(id) as number];
+          const regexp = assertIdentifiers[id];
+          if (!regexp.test(name)) {
+            throw Error(`${description}: The matching identifier "${id}" is "${
+                name}" which doesn't match ${regexp}`);
+          }
+        }
+      }
+    }
+  }
+}
+
+const IDENT_LIKE = /^[a-z][A-Z]/;
+const MATCHING_IDENT = /^\$.*\$$/;
+
+/*
+ * Builds a regexp that matches the given `pieces`
+ *
+ * It returns:
+ * - the `regexp` to be used to match the generated code,
+ * - the `groups` which maps `$...$` identifier to their position in the regexp matches.
+ */
+function buildMatcher(pieces: (string|RegExp)[]): {regexp: RegExp, groups: Map<string, number>} {
+  const results: string[] = [];
+  let first = true;
+  let group = 0;
+
+  const groups = new Map<string, number>();
+  for (const piece of pieces) {
+    if (!first)
+      results.push(`\\s${typeof piece === 'string' && IDENT_LIKE.test(piece) ? '+' : '*'}`);
+    first = false;
+    if (typeof piece === 'string') {
+      if (MATCHING_IDENT.test(piece)) {
+        const matchGroup = groups.get(piece);
+        if (!matchGroup) {
+          results.push('(' + IDENTIFIER.source + ')');
+          const newGroup = ++group;
+          groups.set(piece, newGroup);
+        } else {
+          results.push(`\\${matchGroup}`);
+        }
+      } else {
+        results.push(escapeRegExp(piece));
+      }
+    } else {
+      results.push('(?:' + piece.source + ')');
+    }
+  }
+  return {
+    regexp: new RegExp(results.join('')),
+    groups,
+  };
+}

packages/compiler-cli/test/compliance/test_helpers/get_compliance_tests.ts

@@ -0,0 +1,207 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {AbsoluteFsPath, NodeJSFileSystem, PathSegment} from '../../../src/ngtsc/file_system';
+
+const fs = new NodeJSFileSystem();
+const basePath = fs.resolve(__dirname, '../test_cases');
+
+/**
+ * Search the `test_cases` directory, in the real file-system, for all the compliance tests.
+ *
+ * Test are indicated by a `TEST_CASES.json` file which contains one or more test cases.
+ */
+export function* getAllComplianceTests(): Generator<ComplianceTest> {
+  const testConfigPaths = collectPaths(basePath, segment => segment === 'TEST_CASES.json');
+  for (const testConfigPath of testConfigPaths) {
+    yield* getComplianceTests(testConfigPath);
+  }
+}
+
+/**
+ * Extract all the compliance tests from the TEST_CASES.json file at the `testConfigPath`.
+ *
+ * @param testConfigPath The path, relative to the `test_cases` basePath, of the `TEST_CASES.json`
+ *     config file.
+ */
+export function* getComplianceTests(testConfigPath: string): Generator<ComplianceTest> {
+  const absTestConfigPath = fs.resolve(basePath, testConfigPath);
+  const realTestPath = fs.dirname(absTestConfigPath);
+  const testConfigJSON = JSON.parse(fs.readFile(absTestConfigPath)).cases;
+  const testConfig = Array.isArray(testConfigJSON) ? testConfigJSON : [testConfigJSON];
+  for (const test of testConfig) {
+    const inputFiles = getStringArrayOrDefault(test, 'inputFiles', realTestPath, ['test.ts']);
+    yield {
+      relativePath: fs.relative(basePath, realTestPath),
+      realTestPath,
+      description: getStringOrFail(test, 'description', realTestPath),
+      inputFiles,
+      expectations: parseExpectations(test.expectations, realTestPath, inputFiles),
+      compilerOptions: getConfigOptions(test, 'compilerOptions', realTestPath),
+      angularCompilerOptions: getConfigOptions(test, 'angularCompilerOptions', realTestPath),
+      focusTest: test.focusTest,
+      excludeTest: test.excludeTest,
+    };
+  }
+}
+
+/**
+ * Search the file-system from the `current` path to find all paths that satisfy the `predicate`.
+ */
+function*
+    collectPaths(current: AbsoluteFsPath, predicate: (segment: PathSegment) => boolean):
+        Generator<AbsoluteFsPath> {
+  if (!fs.exists(current)) {
+    return;
+  }
+  for (const segment of fs.readdir(current)) {
+    const absPath = fs.resolve(current, segment);
+    if (predicate(segment)) {
+      yield absPath;
+    } else {
+      if (fs.lstat(absPath).isDirectory()) {
+        yield* collectPaths(absPath, predicate);
+      }
+    }
+  }
+}
+
+function getStringOrFail(container: any, property: string, testPath: AbsoluteFsPath): string {
+  const value = container[property];
+  if (typeof value !== 'string') {
+    throw new Error(`Test is missing "${property}" property in TEST_CASES.json: ` + testPath);
+  }
+  return value;
+}
+
+function getStringArrayOrDefault(
+    container: any, property: string, testPath: AbsoluteFsPath, defaultValue: string[]): string[] {
+  const value = container[property];
+  if (typeof value === 'undefined') {
+    return defaultValue;
+  }
+  if (!Array.isArray(value) || !value.every(item => typeof item === 'string')) {
+    throw new Error(
+        `Test has invalid "${property}" property in TEST_CASES.json - expected array of strings: ` +
+        testPath);
+  }
+  return value;
+}
+
+function parseExpectations(
+    value: any, testPath: AbsoluteFsPath, inputFiles: string[]): Expectation[] {
+  const defaultFailureMessage = 'Incorrect generated output.';
+  const tsFiles = inputFiles.filter(f => /[^.][^d]\.ts$/.test(f));
+  const defaultFiles = tsFiles.map(inputFile => {
+    const outputFile = inputFile.replace(/\.ts$/, '.js');
+    return {expected: outputFile, generated: outputFile};
+  });
+
+  if (typeof value === 'undefined') {
+    return [{failureMessage: defaultFailureMessage, files: defaultFiles}];
+  }
+
+  if (!Array.isArray(value)) {
+    return parseExpectations([value], testPath, inputFiles);
+  }
+
+  return value.map((expectation, i) => {
+    if (typeof expectation !== 'object') {
+      throw new Error(
+          `Test has invalid "expectations" property in TEST_CASES.json - expected array of "expectation" objects: ${
+              testPath}`);
+    }
+
+    const failureMessage = expectation.failureMessage ?? defaultFailureMessage;
+
+    if (typeof expectation.files === 'undefined') {
+      return {failureMessage, files: defaultFiles};
+    }
+
+    if (!Array.isArray(expectation.files)) {
+      throw new Error(`Test has invalid "expectations[${
+          i}].files" property in TEST_CASES.json - expected array of "expected files": ${
+          testPath}`);
+    }
+    const files = expectation.files.map((file: any) => {
+      if (typeof file === 'string') {
+        return {expected: file, generated: file};
+      }
+      if (typeof file === 'object' && typeof file.expected === 'string' &&
+          typeof file.generated === 'string') {
+        return file;
+      }
+      throw new Error(`Test has invalid "expectations[${
+          i}].files" property in TEST_CASES.json - expected each item to be a string or an "expected file" object: ${
+          testPath}`);
+    });
+
+    return {failureMessage, files};
+  });
+}
+
+function getConfigOptions(
+    container: any, property: string, testPath: AbsoluteFsPath): ConfigOptions|undefined {
+  const options = container[property];
+  if (options !== undefined && typeof options !== 'object') {
+    throw new Error(
+        `Test have invalid "${
+            property}" property in TEST_CASES.json - expected config option object: ` +
+        testPath);
+  }
+  return options;
+}
+
+/**
+ * Describes a compliance test, as defined in a `TEST_CASES.json` file.
+ */
+export interface ComplianceTest {
+  /** The path, relative to the test_cases directory, of the directory containing this test. */
+  relativePath: string;
+  /** The absolute path (on the real file-system) to the test case containing this test. */
+  realTestPath: AbsoluteFsPath;
+  /** A description of this particular test. */
+  description: string;
+  /**
+   * Any additional options to pass to the TypeScript compiler when compiling this test's source
+   * files. These are equivalent to what you would put in `tsconfig.json`.
+   */
+  compilerOptions?: ConfigOptions;
+  /**
+   * Any additional options to pass to the Angular compiler when compiling this test's source
+   * files. These are equivalent to what you would put in `tsconfig.json`.
+   */
+  angularCompilerOptions?: ConfigOptions;
+  /** A list of paths to source files that should be compiled for this test case. */
+  inputFiles: string[];
+  /** A list of expectations to check for this test case. */
+  expectations: Expectation[];
+  /** If set to `true`, then focus on this test (equivalent to jasmine's 'fit()`). */
+  focusTest?: boolean;
+  /** If set to `true`, then exclude this test (equivalent to jasmine's 'xit()`). */
+  excludeTest?: boolean;
+}
+
+export interface Expectation {
+  /** The message to display if this expectation fails. */
+  failureMessage: string;
+  /** A list of pairs of paths to expected and generated files to compare. */
+  files: ExpectedFile[];
+}
+
+/**
+ * A pair of paths to expected and generated files that should be compared in an `Expectation`.
+ */
+export interface ExpectedFile {
+  expected: string;
+  generated: string;
+}
+
+/**
+ * Options to pass to configure the compiler.
+ */
+export type ConfigOptions = Record<string, string|boolean|null>;

packages/compiler-cli/test/compliance/test_helpers/test_runner.ts

@@ -0,0 +1,34 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {FileSystem} from '../../../src/ngtsc/file_system';
+import {checkExpectations} from '../test_helpers/check_expectations';
+import {initMockTestFileSystem} from '../test_helpers/compile_test';
+import {ComplianceTest, getAllComplianceTests} from '../test_helpers/get_compliance_tests';
+
+/**
+ * Set up jasmine specs for each of the compliance tests.
+ *
+ * @param type A description of the type of tests being run.
+ * @param compileFn The function that will do the compilation of the source files
+ */
+export function runTests(type: string, compileFn: (fs: FileSystem, test: ComplianceTest) => void) {
+  describe(`compliance tests (${type})`, () => {
+    for (const test of getAllComplianceTests()) {
+      describe(`[${test.relativePath}]`, () => {
+        const itFn = test.focusTest ? fit : test.excludeTest ? xit : it;
+        itFn(test.description, () => {
+          const fs = initMockTestFileSystem(test.realTestPath);
+          compileFn(fs, test);
+          for (const expectation of test.expectations) {
+            checkExpectations(fs, test.relativePath, expectation.failureMessage, expectation.files);
+          }
+        });
+      });
+    }
+  });
+}

tslint.json

@@ -92,6 +92,7 @@
       // Ignore special files
       "**/*.externs.js",
       // Ignore test files
+      "./packages/compiler-cli/test/compliance/test_cases/**/*",
       "./packages/localize/**/test_files/**/*",
       "./tools/ts-api-guardian/test/fixtures/**/*",
       "./tools/public_api_guard/**/*.d.ts",