From 793d66afa5ede1a91eb72bfd44848f94aec9ad35 Mon Sep 17 00:00:00 2001 From: Pete Bacon Darwin Date: Wed, 11 Nov 2020 15:29:43 +0000 Subject: [PATCH] test(compiler-cli): create "full compile" compliance test rules (#39617) This commit contains the basic runner logic and a couple of sample test cases for the "full compile" compliance tests, where source files are compiled to full definitions and checked against expectations. PR Close #39617 --- .ng-dev/format.ts | 2 + .../compiler-cli/test/compliance/README.md | 157 +++++++++++++ .../test/compliance/full/BUILD.bazel | 27 +++ .../test/compliance/full/full_compile_spec.ts | 23 ++ .../test/compliance/test_cases/BUILD.bazel | 11 + .../interpolations/GOLDEN_PARTIAL.js | 47 ++++ .../interpolations/TEST_CASES.json | 13 ++ .../r3_view_compiler/interpolations/test.js | 11 + .../r3_view_compiler/interpolations/test.ts | 14 ++ .../directives/matching/GOLDEN_PARTIAL.js | 50 +++++ .../directives/matching/TEST_CASES.json | 28 +++ .../directives/matching/component.js | 12 + .../directives/matching/factory.js | 3 + .../directives/matching/test.ts | 13 ++ .../test_cases/test_case_schema.json | 89 ++++++++ .../test/compliance/test_helpers/BUILD.bazel | 21 ++ .../test_helpers/check_expectations.ts | 45 ++++ .../compliance/test_helpers/compile_test.ts | 112 ++++++++++ .../compliance/test_helpers/expect_emit.ts | 197 +++++++++++++++++ .../test_helpers/get_compliance_tests.ts | 207 ++++++++++++++++++ .../compliance/test_helpers/test_runner.ts | 34 +++ tslint.json | 1 + 22 files changed, 1117 insertions(+) create mode 100644 packages/compiler-cli/test/compliance/README.md create mode 100644 packages/compiler-cli/test/compliance/full/BUILD.bazel create mode 100644 packages/compiler-cli/test/compliance/full/full_compile_spec.ts create mode 100644 packages/compiler-cli/test/compliance/test_cases/BUILD.bazel create mode 100644 packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/GOLDEN_PARTIAL.js create mode 100644 packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/TEST_CASES.json create mode 100644 packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/test.js create mode 100644 packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/test.ts create mode 100644 packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/GOLDEN_PARTIAL.js create mode 100644 packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/TEST_CASES.json create mode 100644 packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/component.js create mode 100644 packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/factory.js create mode 100644 packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/test.ts create mode 100644 packages/compiler-cli/test/compliance/test_cases/test_case_schema.json create mode 100644 packages/compiler-cli/test/compliance/test_helpers/BUILD.bazel create mode 100644 packages/compiler-cli/test/compliance/test_helpers/check_expectations.ts create mode 100644 packages/compiler-cli/test/compliance/test_helpers/compile_test.ts create mode 100644 packages/compiler-cli/test/compliance/test_helpers/expect_emit.ts create mode 100644 packages/compiler-cli/test/compliance/test_helpers/get_compliance_tests.ts create mode 100644 packages/compiler-cli/test/compliance/test_helpers/test_runner.ts diff --git a/.ng-dev/format.ts b/.ng-dev/format.ts index b75dd04979..e6328eca9c 100644 --- a/.ng-dev/format.ts +++ b/.ng-dev/format.ts @@ -18,6 +18,8 @@ export const format: FormatConfig = { '!**/*.d.ts', // Do not format generated ng-dev script '!dev-infra/ng-dev.js', + // Do not format compliance test-cases since they must match generated code + '!packages/compiler-cli/test/compliance/test_cases/**/*.js', ] }, 'buildifier': true diff --git a/packages/compiler-cli/test/compliance/README.md b/packages/compiler-cli/test/compliance/README.md new file mode 100644 index 0000000000..5315510084 --- /dev/null +++ b/packages/compiler-cli/test/compliance/README.md @@ -0,0 +1,157 @@ +# Compliance test-cases + +This directory contains rules, helpers and test-cases for the Angular compiler compliance tests. + +There are three different types of tests that are run based on file-based "test-cases". + +* **Full compile** - in this test the source files defined by the test-case are fully compiled by Angular. + The generated files are compared to "expected files" via a matching algorithm that is tolerant to + whitespace and variable name changes. +* **Partial compile** - in this test the source files defined by the test-case are "partially" compiled by + Angular to produce files that can be published. These partially compiled files are compared directly + against "golden files" to ensure that we do not inadvertently break the public API of partial + declarations. +* **Linked** - in this test the golden files mentioned in the previous bullet point, are passed to the + Angular linker, which generates files that are comparable to the fully compiled files. These linked + files are compared against the "expected files" in the same way as in the "full compile" tests. + +This way the compliance tests are able to check each mode and stage of compilation is accurate and does +not change unexpectedly. + + +## Defining a test-case + +To define a test-case, create a new directory below `test_cases`. In this directory + +* add a new file called `TEST_CASES.json`. The format of this file is described below. +* add an empty `GOLDEN_PARTIAL.js` file. This file will be updated by the tooling later. +* add any `inputFiles` that will be compiled as part of the test-case. +* add any `expected` files that will be compared to the files generated by compiling the source files. + + +### TEST_CASES.json format + +The `TEST_CASES.json` defines an object with one or more test-case definitions in the `cases` property. + +Each test-case can specify: + +* A `description` of the test. +* The `inputFiles` that will be compiled. +* Additional `compilerOptions` and `angularCompilerOptions` that are passed to the compiler. +* A collection of `expectations` definitions that will be checked against the generated files. + +Note that there is a JSON schema for the `TEST_CASES.json` file stored at `test_cases/test_case_schema.json`. +You should add a link to this schema at the top of your `TEST_CASES.json` file to provide +validation and intellisense for this file in your IDE. + +For example: + +```json +{ + "$schema": "../test_case_schema.json", + "cases": [ + { + "description": "description of the test - equivalent to an `it` clause message.", + "inputFiles": ["abc.ts"], + "expectations": [ + { + "failureMessage": "message to display if this expectation fails", + "files": [ + { "expected": "xyz.js", "generated": "abc.js" }, ... + ] + }, ... + ], + "compilerOptions": { ... }, + "angularCompilerOptions": { ... } + } + ] +} +``` + +### Input files + +The input files are the source file that will be compiled as part of this test-case. +Input files should be stored in the directory next to the `TEST_CASES.json`. +The paths to the input files should be listed in the `inputFiles` property of `TEST_CASES.json`. +The paths are relative to the `TEST_CASES.json` file. + +If no `inputFiles` property is provided, the default is `["test.ts"]`. + + +### Expectations + +An expectation consists of a collection of expected `files` pairs, and a `failureMessage`, which +is displayed if the expectation check fails. + +Each file-pair consists of a path to a `generated` file (relative to the build output folder), +and a path to an `expected` file (relative to the test case). + +The `generated` file is checked to see if it "matches" the `expected` file. The matching is +resilient to whitespace and variable name changes. + +If no `failureMessage` property is provided, the default is `"Incorrect generated output."`. + +If no `files` property is provided, the default is a a collection of objects `{expected, generated}`, +where `expected` and `generated` are computed by taking each path in the `inputFiles` collection +and replacing the `.ts` extension with `.js`. + + +## Running tests + +The simplest way to run all the compliance tests is: + +```sh +yarn test-ivy-aot //packages/compiler-cli/test/compliance/... +``` + +If you only want to run one of the three types of test you can be more specific: + +```sh +yarn test-ivy-aot //packages/compiler-cli/test/compliance/full +yarn test-ivy-aot //packages/compiler-cli/test/compliance/linked +yarn test-ivy-aot //packages/compiler-cli/test/compliance/test_cases/... +``` + +(The last command runs the partial compilation tests.) + + +## Updating a golden partial file + +There is one golden partial file per `TEST_CASES.json` file. So even if this file defines multiple +test-cases, which each contain multiple input files, there will only be one golden file. + +The golden file is generated by the tooling and should not be modified manually. + +When you first create a test-case, with an empty `PARTIAL_GOLDEN.js` file, or a change is made to +the generated partial output, we must update the `PARTIAL_GOLDEN.js` file. + +This is done by running a specific bazel rule of the form: + +```sh +bazel run //packages/compiler-cli/test/compliance/test_cases:.golden.update +``` + +where to replace `` with the path (relative to `test_cases`) of the directory +that contains the `PARTIAL_GOLDEN.js` to update. + + +## Debugging test-cases + +Compliance tests are basically `jasmine_node_test` rules. As such, they can be debugged +just like any other `jasmine_node_test`. The standard approach is to add `--config=debug` +to the Bazel test command. + +It is useful when debugging to focus on a single test-case. + + +### Focusing test-cases + +You can focus a test case by setting `"focusTest": true` in the `TEST_CASES.json` file. +This is equivalent to using jasmine `fit()`. + + +### Excluding test-cases + +You can exclude a test case by setting `"excludeTest": true` in the `TEST_CASES.json` file. +This is equivalent to using jasmine `xit()`. + diff --git a/packages/compiler-cli/test/compliance/full/BUILD.bazel b/packages/compiler-cli/test/compliance/full/BUILD.bazel new file mode 100644 index 0000000000..a8e2fd4e5a --- /dev/null +++ b/packages/compiler-cli/test/compliance/full/BUILD.bazel @@ -0,0 +1,27 @@ +load("//tools:defaults.bzl", "jasmine_node_test", "ts_library") + +ts_library( + name = "test_lib", + testonly = True, + srcs = ["full_compile_spec.ts"], + deps = [ + "//packages/compiler-cli/src/ngtsc/file_system", + "//packages/compiler-cli/test/compliance/test_helpers", + ], +) + +jasmine_node_test( + name = "full", + bootstrap = ["//tools/testing:node_no_angular_es5"], + data = [ + "//packages/compiler-cli/test/compliance/test_cases", + "//packages/compiler-cli/test/ngtsc/fake_core:npm_package", + ], + shard_count = 2, + tags = [ + "ivy-only", + ], + deps = [ + ":test_lib", + ], +) diff --git a/packages/compiler-cli/test/compliance/full/full_compile_spec.ts b/packages/compiler-cli/test/compliance/full/full_compile_spec.ts new file mode 100644 index 0000000000..4e50602e67 --- /dev/null +++ b/packages/compiler-cli/test/compliance/full/full_compile_spec.ts @@ -0,0 +1,23 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {FileSystem} from '../../../src/ngtsc/file_system'; +import {compileTest} from '../test_helpers/compile_test'; +import {ComplianceTest} from '../test_helpers/get_compliance_tests'; +import {runTests} from '../test_helpers/test_runner'; + +runTests('full compile', compileTests); + +/** + * Fully compile all the input files in the given `test`. + * + * @param fs The mock file-system where the input files can be found. + * @param test The compliance test whose input files should be compiled. + */ +function compileTests(fs: FileSystem, test: ComplianceTest): void { + compileTest(fs, test.inputFiles, test.compilerOptions, test.angularCompilerOptions); +} diff --git a/packages/compiler-cli/test/compliance/test_cases/BUILD.bazel b/packages/compiler-cli/test/compliance/test_cases/BUILD.bazel new file mode 100644 index 0000000000..c69d46110a --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/BUILD.bazel @@ -0,0 +1,11 @@ +load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin") + +package(default_visibility = ["//packages/compiler-cli/test/compliance:__subpackages__"]) + +# Normally we would use `file_group` here but this doesn't work on Windows because there +# appears to be a bug with making the `runfiles` folder available. +# See https://github.com/bazelbuild/rules_nodejs/issues/1689 +copy_to_bin( + name = "test_cases", + srcs = glob(["*/**"]), +) diff --git a/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/GOLDEN_PARTIAL.js b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/GOLDEN_PARTIAL.js new file mode 100644 index 0000000000..2f592047b2 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/GOLDEN_PARTIAL.js @@ -0,0 +1,47 @@ +/**************************************************************************************************** + * PARTIAL FILE: test.js + ****************************************************************************************************/ +import { Component, NgModule } from '@angular/core'; +import * as i0 from "@angular/core"; +export class MyApp { + constructor() { + this.list = []; + } +} +MyApp.ɵfac = function MyApp_Factory(t) { return new (t || MyApp)(); }; +MyApp.ɵcmp = i0.ɵɵdefineComponent({ type: MyApp, selectors: [["my-app"]], decls: 1, vars: 9, template: function MyApp_Template(rf, ctx) { if (rf & 1) { + i0.ɵɵtext(0); + } if (rf & 2) { + i0.ɵɵtextInterpolateV([" ", ctx.list[0], " ", ctx.list[1], " ", ctx.list[2], " ", ctx.list[3], " ", ctx.list[4], " ", ctx.list[5], " ", ctx.list[6], " ", ctx.list[7], " ", ctx.list[8], " "]); + } }, encapsulation: 2 }); +/*@__PURE__*/ (function () { i0.ɵsetClassMetadata(MyApp, [{ + type: Component, + args: [{ + selector: 'my-app', + template: ' {{list[0]}} {{list[1]}} {{list[2]}} {{list[3]}} {{list[4]}} {{list[5]}} {{list[6]}} {{list[7]}} {{list[8]}} ' + }] + }], null, null); })(); +export class MyModule { +} +MyModule.ɵmod = i0.ɵɵdefineNgModule({ type: MyModule }); +MyModule.ɵinj = i0.ɵɵdefineInjector({ factory: function MyModule_Factory(t) { return new (t || MyModule)(); } }); +(function () { (typeof ngJitMode === "undefined" || ngJitMode) && i0.ɵɵsetNgModuleScope(MyModule, { declarations: [MyApp] }); })(); +/*@__PURE__*/ (function () { i0.ɵsetClassMetadata(MyModule, [{ + type: NgModule, + args: [{ declarations: [MyApp] }] + }], null, null); })(); + +/**************************************************************************************************** + * PARTIAL FILE: test.d.ts + ****************************************************************************************************/ +import * as i0 from "@angular/core"; +export declare class MyApp { + list: any[]; + static ɵfac: i0.ɵɵFactoryDef; + static ɵcmp: i0.ɵɵComponentDefWithMeta; +} +export declare class MyModule { + static ɵmod: i0.ɵɵNgModuleDefWithMeta; + static ɵinj: i0.ɵɵInjectorDef; +} + diff --git a/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/TEST_CASES.json b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/TEST_CASES.json new file mode 100644 index 0000000000..a238c8f973 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/TEST_CASES.json @@ -0,0 +1,13 @@ +{ + "$schema": "../test_case_schema.json", + "cases": [ + { + "description": "should generate a correct call to `ɵɵtextInterpolateV()` with more than 8 interpolations", + "expectations": [ + { + "failureMessage": "Incorrect `ɵɵtextInterpolateV()` call" + } + ] + } + ] +} diff --git a/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/test.js b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/test.js new file mode 100644 index 0000000000..4701aba609 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/test.js @@ -0,0 +1,11 @@ +function MyApp_Template(rf, ctx) { + if (rf & 1) { + $i0$.ɵɵtext(0); + } + if (rf & 2) { + $i0$.ɵɵtextInterpolateV([ + " ", ctx.list[0], " ", ctx.list[1], " ", ctx.list[2], " ", ctx.list[3], " ", ctx.list[4], " ", + ctx.list[5], " ", ctx.list[6], " ", ctx.list[7], " ", ctx.list[8], " " + ]); + } +} \ No newline at end of file diff --git a/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/test.ts b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/test.ts new file mode 100644 index 0000000000..8d66fbda08 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler/interpolations/test.ts @@ -0,0 +1,14 @@ +import {Component, NgModule} from '@angular/core'; + +@Component({ + selector: 'my-app', + template: + ' {{list[0]}} {{list[1]}} {{list[2]}} {{list[3]}} {{list[4]}} {{list[5]}} {{list[6]}} {{list[7]}} {{list[8]}} ' +}) +export class MyApp { + list: any[] = []; +} + +@NgModule({declarations: [MyApp]}) +export class MyModule { +} \ No newline at end of file diff --git a/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/GOLDEN_PARTIAL.js b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/GOLDEN_PARTIAL.js new file mode 100644 index 0000000000..e3ce742731 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/GOLDEN_PARTIAL.js @@ -0,0 +1,50 @@ +/**************************************************************************************************** + * PARTIAL FILE: test.js + ****************************************************************************************************/ +import { Component, Directive, NgModule } from '@angular/core'; +import * as i0 from "@angular/core"; +export class I18nDirective { +} +I18nDirective.ɵfac = function I18nDirective_Factory(t) { return new (t || I18nDirective)(); }; +I18nDirective.ɵdir = i0.ɵɵngDeclareDirective({ version: 1, type: I18nDirective, selector: "[i18n]", ngImport: i0 }); +/*@__PURE__*/ (function () { i0.ɵsetClassMetadata(I18nDirective, [{ + type: Directive, + args: [{ selector: '[i18n]' }] + }], null, null); })(); +export class MyComponent { +} +MyComponent.ɵfac = function MyComponent_Factory(t) { return new (t || MyComponent)(); }; +MyComponent.ɵcmp = i0.ɵɵdefineComponent({ type: MyComponent, selectors: [["my-component"]], decls: 1, vars: 0, template: function MyComponent_Template(rf, ctx) { if (rf & 1) { + i0.ɵɵelement(0, "div"); + } }, encapsulation: 2 }); +/*@__PURE__*/ (function () { i0.ɵsetClassMetadata(MyComponent, [{ + type: Component, + args: [{ selector: 'my-component', template: '
' }] + }], null, null); })(); +export class MyModule { +} +MyModule.ɵmod = i0.ɵɵdefineNgModule({ type: MyModule }); +MyModule.ɵinj = i0.ɵɵdefineInjector({ factory: function MyModule_Factory(t) { return new (t || MyModule)(); } }); +(function () { (typeof ngJitMode === "undefined" || ngJitMode) && i0.ɵɵsetNgModuleScope(MyModule, { declarations: [I18nDirective, MyComponent] }); })(); +/*@__PURE__*/ (function () { i0.ɵsetClassMetadata(MyModule, [{ + type: NgModule, + args: [{ declarations: [I18nDirective, MyComponent] }] + }], null, null); })(); + +/**************************************************************************************************** + * PARTIAL FILE: test.d.ts + ****************************************************************************************************/ +import * as i0 from "@angular/core"; +export declare class I18nDirective { + static ɵfac: i0.ɵɵFactoryDef; + static ɵdir: i0.ɵɵDirectiveDefWithMeta; +} +export declare class MyComponent { + static ɵfac: i0.ɵɵFactoryDef; + static ɵcmp: i0.ɵɵComponentDefWithMeta; +} +export declare class MyModule { + static ɵmod: i0.ɵɵNgModuleDefWithMeta; + static ɵinj: i0.ɵɵInjectorDef; +} + diff --git a/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/TEST_CASES.json b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/TEST_CASES.json new file mode 100644 index 0000000000..a693113109 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/TEST_CASES.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../test_case_schema.json", + "cases": [ + { + "description": "should not match directives on i18n attribute", + "expectations": [ + { + "failureMessage": "Incorrect ChildComponent.ɵcmp", + "files": [ + { + "expected": "component.js", + "generated": "test.js" + } + ] + }, + { + "failureMessage": "Incorrect ChildComponent.ɵfac", + "files": [ + { + "expected": "factory.js", + "generated": "test.js" + } + ] + } + ] + } + ] +} diff --git a/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/component.js b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/component.js new file mode 100644 index 0000000000..42bf7ae2fa --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/component.js @@ -0,0 +1,12 @@ +MyComponent.ɵcmp = $r3$.ɵɵdefineComponent({ + type: MyComponent, + selectors: [["my-component"]], + decls: 1, + vars: 0, + template: function MyComponent_Template(rf, ctx) { + if (rf & 1) { + $r3$.ɵɵelement(0, "div"); + } + }, + encapsulation: 2 +}); \ No newline at end of file diff --git a/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/factory.js b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/factory.js new file mode 100644 index 0000000000..4348030bd5 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/factory.js @@ -0,0 +1,3 @@ +MyComponent.ɵfac = function MyComponent_Factory(t) { + return new (t || MyComponent)(); +}; \ No newline at end of file diff --git a/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/test.ts b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/test.ts new file mode 100644 index 0000000000..875b9e7c48 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_directives/directives/matching/test.ts @@ -0,0 +1,13 @@ +import {Component, Directive, NgModule} from '@angular/core'; + +@Directive({selector: '[i18n]'}) +export class I18nDirective { +} + +@Component({selector: 'my-component', template: '
'}) +export class MyComponent { +} + +@NgModule({declarations: [I18nDirective, MyComponent]}) +export class MyModule { +} diff --git a/packages/compiler-cli/test/compliance/test_cases/test_case_schema.json b/packages/compiler-cli/test/compliance/test_cases/test_case_schema.json new file mode 100644 index 0000000000..22cfe7e179 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_cases/test_case_schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://angular.io/internal/compliance-test-cases.json", + "title": "Compliance test cases", + "description": "Describes configuration for an Angular compliance test", + "type": "object", + "properties": { + "cases": { + "title": "A collection of test cases", + "type": "array", + "items": { + "title": "A test case definition", + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "title": "A description of the test case", + "description": "This will be used as the message in an `it()` clause.", + "type": "string" + }, + "inputFiles": { + "title": "A collection of source files to compile", + "type": "array", + "items": { + "title": "Path (relative to the test case) of a source file to compile", + "type": "string" + } + }, + "expectations": { + "title": "A collection of expectations for this test case", + "type": "array", + "items": { + "title": "An expectation to check for this test case", + "type": "object", + "properties": { + "failureMessage": { + "title": "The message to display if this expectation fails", + "type": "string", + "default": "Incorrect generated output." + }, + "files": { + "title": "A collection of expected-generated file path pairs", + "type": "array", + "items": { + "title": "A pair of expected-generated file paths to be compared", + "type": "object", + "required": [ + "expected", + "generated" + ], + "properties": { + "expected": { + "title": "A path (relative to the test case) where an file containing expected output can be found", + "type": "string" + }, + "generated": { + "title": "A path (relative to the build output directory) where the compiled file can be found.", + "type": "string" + } + } + } + } + } + } + }, + "compilerOptions": { + "title": "Additional options to pass to the TypeScript compiler", + "type": "object" + }, + "angularCompilerOptions": { + "title": "Additional options to pass to the Angular compiler", + "type": "object" + }, + "focusTest": { + "title": "Set to true to focus on this test - equivalent to jasmine's `fit()` function", + "type": "boolean" + }, + "excludeTest": { + "title": "Set to true to exclude this test - equivalent to jasmine's `xit()` function", + "type": "boolean" + } + } + }, + "minItems": 1 + } + } +} diff --git a/packages/compiler-cli/test/compliance/test_helpers/BUILD.bazel b/packages/compiler-cli/test/compliance/test_helpers/BUILD.bazel new file mode 100644 index 0000000000..bf4e4af039 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_helpers/BUILD.bazel @@ -0,0 +1,21 @@ +load("//tools:defaults.bzl", "ts_library") + +ts_library( + name = "test_helpers", + testonly = True, + srcs = glob( + ["**/*.ts"], + ), + visibility = [ + "//packages/compiler-cli/test/compliance:__subpackages__", + ], + deps = [ + "//packages:types", + "//packages/compiler", + "//packages/compiler-cli", + "//packages/compiler-cli/src/ngtsc/file_system", + "//packages/compiler-cli/src/ngtsc/file_system/testing", + "//packages/compiler-cli/test/helpers", + "@npm//typescript", + ], +) diff --git a/packages/compiler-cli/test/compliance/test_helpers/check_expectations.ts b/packages/compiler-cli/test/compliance/test_helpers/check_expectations.ts new file mode 100644 index 0000000000..c80d194702 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_helpers/check_expectations.ts @@ -0,0 +1,45 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {FileSystem} from '../../../src/ngtsc/file_system'; +import {getBuildOutputDirectory} from './compile_test'; +import {expectEmit} from './expect_emit'; +import {ExpectedFile} from './get_compliance_tests'; + +/** + * Check that each of the generated files matches the expected files. + * + * @param fs The mock file-system that holds the expected and generated files to compare. + * @param testPath Path to the current test case (relative to the basePath). + * @param failureMessage The message to display if the expectation fails. + * @param expectedFiles The list of expected-generated pairs to compare. + */ +export function checkExpectations( + fs: FileSystem, testPath: string, failureMessage: string, expectedFiles: ExpectedFile[]): void { + const builtDirectory = getBuildOutputDirectory(fs); + for (const expectedFile of expectedFiles) { + const expectedPath = fs.resolve(expectedFile.expected); + if (!fs.exists(expectedPath)) { + throw new Error(`The expected file at ${ + expectedPath} does not exist. Please check the TEST_CASES.json file for this test case.`); + } + + const generatedPath = fs.resolve(builtDirectory, expectedFile.generated); + if (!fs.exists(generatedPath)) { + throw new Error(`The generated file at ${ + generatedPath} does not exist. Perhaps there is no matching input source file in the TEST_CASES.json file for this test case.`); + } + + const expected = fs.readFile(expectedPath); + const generated = fs.readFile(generatedPath); + + expectEmit( + generated, expected, + `When checking against expected file "${testPath}/${expectedFile.expected}"\n` + + failureMessage); + } +} diff --git a/packages/compiler-cli/test/compliance/test_helpers/compile_test.ts b/packages/compiler-cli/test/compliance/test_helpers/compile_test.ts new file mode 100644 index 0000000000..3b96d635ff --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_helpers/compile_test.ts @@ -0,0 +1,112 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import * as ts from 'typescript'; + +import {AbsoluteFsPath, FileSystem, NgtscCompilerHost} from '../../../src/ngtsc/file_system'; +import {initMockFileSystem} from '../../../src/ngtsc/file_system/testing'; +import {performCompilation} from '../../../src/perform_compile'; +import {CompilerOptions} from '../../../src/transformers/api'; +import {loadStandardTestFiles, loadTestDirectory} from '../../helpers'; + +import {ConfigOptions} from './get_compliance_tests'; + +/** + * Setup a mock file-system that is used to generate the partial files. + * + * @param realTestPath Absolute path (on the real file-system) to the test case being processed. + * @returns a mock file-system containing the test case files. + */ +export function initMockTestFileSystem(realTestPath: AbsoluteFsPath): FileSystem { + const fs = initMockFileSystem('Native'); + const testFiles = loadStandardTestFiles(); + fs.init(testFiles); + loadTestDirectory(fs, realTestPath, getRootDirectory(fs)); + return fs; +} + +/** + * Compile the input source `files` stored in `fs`, writing the generated files to `fs`. + * + * @param fs The mock file-system where the input and generated files live. + * @param files An array of paths (relative to the testPath) of input files to be compiled. + * @param compilerOptions Any extra options to pass to the TypeScript compiler. + * @param angularCompilerOptions Any extra options to pass to the Angular compiler. + * @returns A collection of paths of the generated files (absolute within the mock file-system). + */ +export function compileTest( + fs: FileSystem, files: string[], compilerOptions: ConfigOptions|undefined, + angularCompilerOptions: ConfigOptions|undefined): AbsoluteFsPath[] { + const rootDir = getRootDirectory(fs); + const outDir = getBuildOutputDirectory(fs); + const options = getOptions(rootDir, outDir, compilerOptions, angularCompilerOptions); + const rootNames = files.map(f => fs.resolve(f)); + const host = new NgtscCompilerHost(fs, options); + const {diagnostics, emitResult} = performCompilation({rootNames, host, options}); + if (diagnostics.length > 0) { + console.warn(diagnostics.map(d => d.messageText).join('\n')); + } + return emitResult!.emittedFiles!.map(p => fs.resolve(rootDir, p)); +} + +/** + * Gets an absolute path (in the mock file-system) of the root directory where the compilation is to + * be done. + * + * @param fs the mock file-system where the compilation is happening. + */ +export function getRootDirectory(fs: FileSystem): AbsoluteFsPath { + return fs.resolve('/'); +} + +/** + * Gets an absolute path (in the mock file-system) of the directory where the compiled files are + * stored. + * + * @param fs the mock file-system where the compilation is happening. + */ +export function getBuildOutputDirectory(fs: FileSystem): AbsoluteFsPath { + return fs.resolve('/built'); +} + +/** + * Get the options object to pass to the compiler. + * + * @param rootDir The absolute path (within the mock file-system) that is the root of the + * compilation. + * @param outDir The absolute path (within the mock file-system) where compiled files will be + * written. + * @param compilerOptions Additional options for the TypeScript compiler. + * @param angularCompilerOptions Additional options for the Angular compiler. + */ +function getOptions( + rootDir: AbsoluteFsPath, outDir: AbsoluteFsPath, compilerOptions: ConfigOptions|undefined, + angularCompilerOptions: ConfigOptions|undefined): CompilerOptions { + return { + emitDecoratorMetadata: true, + experimentalDecorators: true, + skipLibCheck: true, + noImplicitAny: true, + noEmitOnError: true, + listEmittedFiles: true, + strictNullChecks: true, + outDir, + rootDir, + baseUrl: '.', + allowJs: true, + declaration: true, + target: ts.ScriptTarget.ES2015, + newLine: ts.NewLineKind.LineFeed, + module: ts.ModuleKind.ES2015, + moduleResolution: ts.ModuleResolutionKind.NodeJs, + typeRoots: ['node_modules/@types'], + ...ts.convertCompilerOptionsFromJson({compilerOptions}, rootDir).options, + enableIvy: true, + ivyTemplateTypeCheck: false, + ...angularCompilerOptions, + }; +} diff --git a/packages/compiler-cli/test/compliance/test_helpers/expect_emit.ts b/packages/compiler-cli/test/compliance/test_helpers/expect_emit.ts new file mode 100644 index 0000000000..1c41c4ecda --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_helpers/expect_emit.ts @@ -0,0 +1,197 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {escapeRegExp} from '@angular/compiler/src/util'; + +const IDENTIFIER = /[A-Za-z_$ɵ][A-Za-z0-9_$]*/; +const OPERATOR = + /!|\?|%|\*|\/|\^|&&?|\|\|?|\(|\)|\{|\}|\[|\]|:|;|<=?|>=?|={1,3}|!==?|=>|\+\+?|--?|@|,|\.|\.\.\.|`|\\'/; +const STRING = /'(\\'|[^'])*'|"(\\"|[^"])*"/; +const BACKTICK_STRING = /\\`(([\s\S]*?)(\$\{[^}]*?\})?)*?[^\\]\\`/; +const BACKTICK_INTERPOLATION = /(\$\{[^}]*\})/; +const NUMBER = /\d+/; + +const ELLIPSIS = '…'; +const TOKEN = new RegExp( + `\\s*((${IDENTIFIER.source})|(${BACKTICK_STRING.source})|(${OPERATOR.source})|(${ + STRING.source})|${NUMBER.source}|${ELLIPSIS})\\s*`, + 'y'); + +type Piece = string|RegExp; + +const SKIP = /(?:.|\n|\r)*/; + +const ERROR_CONTEXT_WIDTH = 30; +// Transform the expected output to set of tokens +function tokenize(text: string): Piece[] { + // TOKEN.lastIndex is stateful so we cache the `lastIndex` and restore it at the end of the call. + const lastIndex = TOKEN.lastIndex; + TOKEN.lastIndex = 0; + + let match: RegExpMatchArray|null; + let tokenizedTextEnd = 0; + const pieces: Piece[] = []; + + while ((match = TOKEN.exec(text)) !== null) { + const [fullMatch, token] = match; + if (token === 'IDENT') { + pieces.push(IDENTIFIER); + } else if (token === ELLIPSIS) { + pieces.push(SKIP); + } else if (match = BACKTICK_STRING.exec(token)) { + pieces.push(...tokenizeBackTickString(token)); + } else { + pieces.push(token); + } + tokenizedTextEnd += fullMatch.length; + } + + if (pieces.length === 0 || tokenizedTextEnd < text.length) { + // The new token that could not be found is located after the + // last tokenized character. + const from = tokenizedTextEnd; + const to = from + ERROR_CONTEXT_WIDTH; + throw Error( + `Invalid test, no token found for "${text[tokenizedTextEnd]}" ` + + `(context = '${text.substr(from, to)}...'`); + } + // Reset the lastIndex in case we are in a recursive `tokenize()` call. + TOKEN.lastIndex = lastIndex; + + return pieces; +} + +/** + * Back-ticks are escaped as "\`" so we must strip the backslashes. + * Also the string will likely contain interpolations and if an interpolation holds an + * identifier we will need to match that later. So tokenize the interpolation too! + */ +function tokenizeBackTickString(str: string): Piece[] { + const pieces: Piece[] = ['`']; + // Unescape backticks that are inside the backtick string + // (we had to double escape them in the test string so they didn't look like string markers) + str = str.replace(/\\\\\\`/g, '\\`'); + const backTickPieces = str.slice(2, -2).split(BACKTICK_INTERPOLATION); + backTickPieces.forEach((backTickPiece) => { + if (BACKTICK_INTERPOLATION.test(backTickPiece)) { + // An interpolation so tokenize this expression + pieces.push(...tokenize(backTickPiece)); + } else { + // Not an interpolation so just add it as a piece + pieces.push(backTickPiece); + } + }); + pieces.push('`'); + return pieces; +} + +export function expectEmit( + source: string, expected: string, description: string, + assertIdentifiers?: {[name: string]: RegExp}) { + expected = expected + // turns `// ...` into `…` + .replace(/\/\/\s*\.\.\./g, ELLIPSIS) + // remove `// TODO` comment lines + .replace(/\/\/\s*TODO.*?\n/g, '') + // remove `// NOTE` comment lines + .replace(/\/\/\s*NOTE.*?\n/g, ''); + + const pieces = tokenize(expected); + const {regexp, groups} = buildMatcher(pieces); + const matches = source.match(regexp); + if (matches === null) { + let last: number = 0; + for (let i = 1; i < pieces.length; i++) { + const {regexp} = buildMatcher(pieces.slice(0, i)); + const m = source.match(regexp); + const expectedPiece = pieces[i - 1] == IDENTIFIER ? '' : pieces[i - 1]; + if (!m) { + // display at most `contextLength` characters of the line preceding the error location + const contextLength = 50; + const fullContext = source.substring(source.lastIndexOf('\n', last) + 1, last); + const context = fullContext.length > contextLength ? + `...${fullContext.substr(-contextLength)}` : + fullContext; + throw new Error( + `${description}:\nFailed to find "${expectedPiece}" after "${context}" in:\n'${ + source.substr( + 0, last)}[<---HERE expected "${expectedPiece}"]${source.substr(last)}'`); + return; + } else { + last = (m.index || 0) + m[0].length; + } + } + throw new Error( + `Test helper failure: Expected expression failed but the reporting logic could not find where it failed in: ${ + source}`); + } else { + if (assertIdentifiers) { + // It might be possible to add the constraints in the original regexp (see `buildMatcher`) + // by transforming the assertion regexps when using anchoring, grouping, back references, + // flags, ... + // + // Checking identifiers after they have matched allows for a simple and flexible + // implementation. + // The overall performance are not impacted when `assertIdentifiers` is empty. + const ids = Object.keys(assertIdentifiers); + for (let i = 0; i < ids.length; i++) { + const id = ids[i]; + if (groups.has(id)) { + const name = matches[groups.get(id) as number]; + const regexp = assertIdentifiers[id]; + if (!regexp.test(name)) { + throw Error(`${description}: The matching identifier "${id}" is "${ + name}" which doesn't match ${regexp}`); + } + } + } + } + } +} + +const IDENT_LIKE = /^[a-z][A-Z]/; +const MATCHING_IDENT = /^\$.*\$$/; + +/* + * Builds a regexp that matches the given `pieces` + * + * It returns: + * - the `regexp` to be used to match the generated code, + * - the `groups` which maps `$...$` identifier to their position in the regexp matches. + */ +function buildMatcher(pieces: (string|RegExp)[]): {regexp: RegExp, groups: Map} { + const results: string[] = []; + let first = true; + let group = 0; + + const groups = new Map(); + for (const piece of pieces) { + if (!first) + results.push(`\\s${typeof piece === 'string' && IDENT_LIKE.test(piece) ? '+' : '*'}`); + first = false; + if (typeof piece === 'string') { + if (MATCHING_IDENT.test(piece)) { + const matchGroup = groups.get(piece); + if (!matchGroup) { + results.push('(' + IDENTIFIER.source + ')'); + const newGroup = ++group; + groups.set(piece, newGroup); + } else { + results.push(`\\${matchGroup}`); + } + } else { + results.push(escapeRegExp(piece)); + } + } else { + results.push('(?:' + piece.source + ')'); + } + } + return { + regexp: new RegExp(results.join('')), + groups, + }; +} diff --git a/packages/compiler-cli/test/compliance/test_helpers/get_compliance_tests.ts b/packages/compiler-cli/test/compliance/test_helpers/get_compliance_tests.ts new file mode 100644 index 0000000000..4e4710727e --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_helpers/get_compliance_tests.ts @@ -0,0 +1,207 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {AbsoluteFsPath, NodeJSFileSystem, PathSegment} from '../../../src/ngtsc/file_system'; + +const fs = new NodeJSFileSystem(); +const basePath = fs.resolve(__dirname, '../test_cases'); + +/** + * Search the `test_cases` directory, in the real file-system, for all the compliance tests. + * + * Test are indicated by a `TEST_CASES.json` file which contains one or more test cases. + */ +export function* getAllComplianceTests(): Generator { + const testConfigPaths = collectPaths(basePath, segment => segment === 'TEST_CASES.json'); + for (const testConfigPath of testConfigPaths) { + yield* getComplianceTests(testConfigPath); + } +} + +/** + * Extract all the compliance tests from the TEST_CASES.json file at the `testConfigPath`. + * + * @param testConfigPath The path, relative to the `test_cases` basePath, of the `TEST_CASES.json` + * config file. + */ +export function* getComplianceTests(testConfigPath: string): Generator { + const absTestConfigPath = fs.resolve(basePath, testConfigPath); + const realTestPath = fs.dirname(absTestConfigPath); + const testConfigJSON = JSON.parse(fs.readFile(absTestConfigPath)).cases; + const testConfig = Array.isArray(testConfigJSON) ? testConfigJSON : [testConfigJSON]; + for (const test of testConfig) { + const inputFiles = getStringArrayOrDefault(test, 'inputFiles', realTestPath, ['test.ts']); + yield { + relativePath: fs.relative(basePath, realTestPath), + realTestPath, + description: getStringOrFail(test, 'description', realTestPath), + inputFiles, + expectations: parseExpectations(test.expectations, realTestPath, inputFiles), + compilerOptions: getConfigOptions(test, 'compilerOptions', realTestPath), + angularCompilerOptions: getConfigOptions(test, 'angularCompilerOptions', realTestPath), + focusTest: test.focusTest, + excludeTest: test.excludeTest, + }; + } +} + +/** + * Search the file-system from the `current` path to find all paths that satisfy the `predicate`. + */ +function* + collectPaths(current: AbsoluteFsPath, predicate: (segment: PathSegment) => boolean): + Generator { + if (!fs.exists(current)) { + return; + } + for (const segment of fs.readdir(current)) { + const absPath = fs.resolve(current, segment); + if (predicate(segment)) { + yield absPath; + } else { + if (fs.lstat(absPath).isDirectory()) { + yield* collectPaths(absPath, predicate); + } + } + } +} + +function getStringOrFail(container: any, property: string, testPath: AbsoluteFsPath): string { + const value = container[property]; + if (typeof value !== 'string') { + throw new Error(`Test is missing "${property}" property in TEST_CASES.json: ` + testPath); + } + return value; +} + +function getStringArrayOrDefault( + container: any, property: string, testPath: AbsoluteFsPath, defaultValue: string[]): string[] { + const value = container[property]; + if (typeof value === 'undefined') { + return defaultValue; + } + if (!Array.isArray(value) || !value.every(item => typeof item === 'string')) { + throw new Error( + `Test has invalid "${property}" property in TEST_CASES.json - expected array of strings: ` + + testPath); + } + return value; +} + +function parseExpectations( + value: any, testPath: AbsoluteFsPath, inputFiles: string[]): Expectation[] { + const defaultFailureMessage = 'Incorrect generated output.'; + const tsFiles = inputFiles.filter(f => /[^.][^d]\.ts$/.test(f)); + const defaultFiles = tsFiles.map(inputFile => { + const outputFile = inputFile.replace(/\.ts$/, '.js'); + return {expected: outputFile, generated: outputFile}; + }); + + if (typeof value === 'undefined') { + return [{failureMessage: defaultFailureMessage, files: defaultFiles}]; + } + + if (!Array.isArray(value)) { + return parseExpectations([value], testPath, inputFiles); + } + + return value.map((expectation, i) => { + if (typeof expectation !== 'object') { + throw new Error( + `Test has invalid "expectations" property in TEST_CASES.json - expected array of "expectation" objects: ${ + testPath}`); + } + + const failureMessage = expectation.failureMessage ?? defaultFailureMessage; + + if (typeof expectation.files === 'undefined') { + return {failureMessage, files: defaultFiles}; + } + + if (!Array.isArray(expectation.files)) { + throw new Error(`Test has invalid "expectations[${ + i}].files" property in TEST_CASES.json - expected array of "expected files": ${ + testPath}`); + } + const files = expectation.files.map((file: any) => { + if (typeof file === 'string') { + return {expected: file, generated: file}; + } + if (typeof file === 'object' && typeof file.expected === 'string' && + typeof file.generated === 'string') { + return file; + } + throw new Error(`Test has invalid "expectations[${ + i}].files" property in TEST_CASES.json - expected each item to be a string or an "expected file" object: ${ + testPath}`); + }); + + return {failureMessage, files}; + }); +} + +function getConfigOptions( + container: any, property: string, testPath: AbsoluteFsPath): ConfigOptions|undefined { + const options = container[property]; + if (options !== undefined && typeof options !== 'object') { + throw new Error( + `Test have invalid "${ + property}" property in TEST_CASES.json - expected config option object: ` + + testPath); + } + return options; +} + +/** + * Describes a compliance test, as defined in a `TEST_CASES.json` file. + */ +export interface ComplianceTest { + /** The path, relative to the test_cases directory, of the directory containing this test. */ + relativePath: string; + /** The absolute path (on the real file-system) to the test case containing this test. */ + realTestPath: AbsoluteFsPath; + /** A description of this particular test. */ + description: string; + /** + * Any additional options to pass to the TypeScript compiler when compiling this test's source + * files. These are equivalent to what you would put in `tsconfig.json`. + */ + compilerOptions?: ConfigOptions; + /** + * Any additional options to pass to the Angular compiler when compiling this test's source + * files. These are equivalent to what you would put in `tsconfig.json`. + */ + angularCompilerOptions?: ConfigOptions; + /** A list of paths to source files that should be compiled for this test case. */ + inputFiles: string[]; + /** A list of expectations to check for this test case. */ + expectations: Expectation[]; + /** If set to `true`, then focus on this test (equivalent to jasmine's 'fit()`). */ + focusTest?: boolean; + /** If set to `true`, then exclude this test (equivalent to jasmine's 'xit()`). */ + excludeTest?: boolean; +} + +export interface Expectation { + /** The message to display if this expectation fails. */ + failureMessage: string; + /** A list of pairs of paths to expected and generated files to compare. */ + files: ExpectedFile[]; +} + +/** + * A pair of paths to expected and generated files that should be compared in an `Expectation`. + */ +export interface ExpectedFile { + expected: string; + generated: string; +} + +/** + * Options to pass to configure the compiler. + */ +export type ConfigOptions = Record; diff --git a/packages/compiler-cli/test/compliance/test_helpers/test_runner.ts b/packages/compiler-cli/test/compliance/test_helpers/test_runner.ts new file mode 100644 index 0000000000..b02abe3797 --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_helpers/test_runner.ts @@ -0,0 +1,34 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {FileSystem} from '../../../src/ngtsc/file_system'; +import {checkExpectations} from '../test_helpers/check_expectations'; +import {initMockTestFileSystem} from '../test_helpers/compile_test'; +import {ComplianceTest, getAllComplianceTests} from '../test_helpers/get_compliance_tests'; + +/** + * Set up jasmine specs for each of the compliance tests. + * + * @param type A description of the type of tests being run. + * @param compileFn The function that will do the compilation of the source files + */ +export function runTests(type: string, compileFn: (fs: FileSystem, test: ComplianceTest) => void) { + describe(`compliance tests (${type})`, () => { + for (const test of getAllComplianceTests()) { + describe(`[${test.relativePath}]`, () => { + const itFn = test.focusTest ? fit : test.excludeTest ? xit : it; + itFn(test.description, () => { + const fs = initMockTestFileSystem(test.realTestPath); + compileFn(fs, test); + for (const expectation of test.expectations) { + checkExpectations(fs, test.relativePath, expectation.failureMessage, expectation.files); + } + }); + }); + } + }); +} diff --git a/tslint.json b/tslint.json index f2ff3e44a3..d7a30f3370 100644 --- a/tslint.json +++ b/tslint.json @@ -92,6 +92,7 @@ // Ignore special files "**/*.externs.js", // Ignore test files + "./packages/compiler-cli/test/compliance/test_cases/**/*", "./packages/localize/**/test_files/**/*", "./tools/ts-api-guardian/test/fixtures/**/*", "./tools/public_api_guard/**/*.d.ts",