test(compiler-cli): improve compliance test compile mode filtering (#39939)

Previously one could set a flag in a `TEST_CASES.json` file to exclude
the test-cases from being run if the input files were being compiled
partially and then linked.

There are also scenarios where one might want to exclude test-cases
from "full compile" mode test runs.

This commit changes the compliance test tooling to support a new
property `compilationModeFilter`, which is an array containing one or
more of `"full compile"` and `"linked compile"`. Only the tests
whose `compilationModeFilter` array contains the current compilation
mode will be run.

PR Close #39939

packages/compiler-cli/test/compliance/README.md

@@ -38,7 +38,7 @@ Each test-case can specify:
 * A `description` of the test.
 * The `inputFiles` that will be compiled.
 * Additional `compilerOptions` and `angularCompilerOptions` that are passed to the compiler.
-* Whether to exclude this test-case from partial compilation tests (`excludeFromPartialTests`).
+* Whether to exclude this test-case from certain tests running under certain compilation modes (`compilationModeFilter`).
 * A collection of `expectations` definitions that will be checked against the generated files.
 
 Note that there is a JSON schema for the `TEST_CASES.json` file stored at `test_cases/test_case_schema.json`.

packages/compiler-cli/test/compliance/linked/linked_compile_spec.ts

@@ -14,7 +14,7 @@ import {ComplianceTest} from '../test_helpers/get_compliance_tests';
 import {parseGoldenPartial} from '../test_helpers/golden_partials';
 import {runTests} from '../test_helpers/test_runner';
 
-runTests('partial compile + link', linkPartials);
+runTests('linked compile', linkPartials);
 
 /**
  * Link all the partials specified in the given `test`.

packages/compiler-cli/test/compliance/test_cases/r3_compiler_compliance/components_and_directives/TEST_CASES.json

@@ -135,7 +135,9 @@
       "compilerOptions": {
         "target": "ES5"
       },
-      "excludeFromPartialTests": true,
+      "compilationModeFilter": [
+        "full compile"
+      ],
       "expectations": [
         {
           "failureMessage": "Incorrect setClassMetadata call",

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_i18n/errors/TEST_CASES.json

@@ -6,7 +6,9 @@
       "inputFiles": [
         "nested_i18n_msg.ts"
       ],
-      "excludeFromPartialTests": true,
+      "compilationModeFilter": [
+        "full compile"
+      ],
       "expectations": [
         {
           "expectedErrors": [
@@ -23,7 +25,9 @@
       "inputFiles": [
         "nested_i18n_msg_with_tags.ts"
       ],
-      "excludeFromPartialTests": true,
+      "compilationModeFilter": [
+        "full compile"
+      ],
       "expectations": [
         {
           "expectedErrors": [
@@ -40,7 +44,9 @@
       "inputFiles": [
         "nested_i18n_msg_with_ng-containers.ts"
       ],
-      "excludeFromPartialTests": true,
+      "compilationModeFilter": [
+        "full compile"
+      ],
       "expectations": [
         {
           "expectedErrors": [

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_i18n/es5_support/TEST_CASES.json

@@ -6,7 +6,9 @@
       "compilerOptions": {
         "target": "ES5"
       },
-      "excludeFromPartialTests": true,
+      "compilationModeFilter": [
+        "full compile"
+      ],
       "expectations": [
         {
           "extraChecks": [

packages/compiler-cli/test/compliance/test_cases/r3_view_compiler_styling/invalid/TEST_CASES.json

@@ -15,7 +15,9 @@
           ]
         }
       ],
-      "excludeFromPartialTests": true
+      "compilationModeFilter": [
+        "full compile"
+      ]
     }
   ]
 }

packages/compiler-cli/test/compliance/test_cases/test_case_schema.json

@@ -20,10 +20,20 @@
             "description": "This will be used as the message in an `it()` clause.",
             "type": "string"
           },
-          "excludeFromPartialTests": {
+          "compilationModeFilter": {
-            "title": "If set to true then do not check this test-case expectations in partial tests.",
+            "title": "An array of compilation modes under which this test-case should be run.",
-            "type": "boolean",
+            "type": "array",
-            "default": false
+            "items": {
+              "type": "string",
+              "enum": [
+                "full compile",
+                "linked compile"
+              ]
+            },
+            "default": [
+              "full compile",
+              "linked compile"
+            ]
           },
           "inputFiles": {
             "title": "A collection of source files to compile",

packages/compiler-cli/test/compliance/test_helpers/get_compliance_tests.ts

@@ -35,13 +35,16 @@ export function* getComplianceTests(testConfigPath: string): Generator<Complianc
   const testConfig = Array.isArray(testConfigJSON) ? testConfigJSON : [testConfigJSON];
   for (const test of testConfig) {
     const inputFiles = getStringArrayOrDefault(test, 'inputFiles', realTestPath, ['test.ts']);
+    const compilationModeFilter = getStringArrayOrDefault(
+                                      test, 'compilationModeFilter', realTestPath,
+                                      ['linked compile', 'full compile']) as CompilationMode[];
+
     yield {
       relativePath: fs.relative(basePath, realTestPath),
       realTestPath,
       description: getStringOrFail(test, 'description', realTestPath),
       inputFiles,
-      excludeFromPartialTests:
+      compilationModeFilter,
-          getBooleanOrDefault(test, 'excludeFromPartialTests', realTestPath, false),
       expectations: parseExpectations(test.expectations, realTestPath, inputFiles),
       compilerOptions: getConfigOptions(test, 'compilerOptions', realTestPath),
       angularCompilerOptions: getConfigOptions(test, 'angularCompilerOptions', realTestPath),
@@ -243,8 +246,11 @@ export interface ComplianceTest {
   angularCompilerOptions?: ConfigOptions;
   /** A list of paths to source files that should be compiled for this test case. */
   inputFiles: string[];
-  /** If set to true then do not check expectations for this test-case in partial tests. */
+  /**
-  excludeFromPartialTests: boolean;
+   * Only run this test when the input files are compiled using the given compilation
+   * modes. The default is to run for all modes.
+   */
+  compilationModeFilter: CompilationMode[];
   /** A list of expectations to check for this test case. */
   expectations: Expectation[];
   /** If set to `true`, then focus on this test (equivalent to jasmine's 'fit()`). */
@@ -253,6 +259,8 @@ export interface ComplianceTest {
   excludeTest?: boolean;
 }
 
+export type CompilationMode = 'linked compile'|'full compile';
+
 export interface Expectation {
   /** The message to display if this expectation fails. */
   failureMessage: string;

packages/compiler-cli/test/compliance/test_helpers/test_runner.ts

@@ -8,7 +8,7 @@
 import {FileSystem} from '../../../src/ngtsc/file_system';
 import {checkExpectations} from '../test_helpers/check_expectations';
 import {CompileResult, initMockTestFileSystem} from '../test_helpers/compile_test';
-import {ComplianceTest, getAllComplianceTests} from '../test_helpers/get_compliance_tests';
+import {CompilationMode, ComplianceTest, getAllComplianceTests} from '../test_helpers/get_compliance_tests';
 import {checkErrors, checkNoUnexpectedErrors} from './check_errors';
 
 /**
@@ -18,23 +18,20 @@ import {checkErrors, checkNoUnexpectedErrors} from './check_errors';
  * @param compileFn The function that will do the compilation of the source files
  */
 export function runTests(
-    type: 'partial compile + link'|'full compile',
+    type: CompilationMode, compileFn: (fs: FileSystem, test: ComplianceTest) => CompileResult) {
-    compileFn: (fs: FileSystem, test: ComplianceTest) => CompileResult) {
-  const isPartial = type === 'partial compile + link';
-
   describe(`compliance tests (${type})`, () => {
     for (const test of getAllComplianceTests()) {
-      if (isPartial && test.excludeFromPartialTests) {
+      if (!test.compilationModeFilter.includes(type)) {
         continue;
       }
 
       describe(`[${test.relativePath}]`, () => {
         const itFn = test.focusTest ? fit : test.excludeTest ? xit : it;
         itFn(test.description, () => {
-          if (isPartial && test.compilerOptions?.target === 'ES5') {
+          if (type === 'linked compile' && test.compilerOptions?.target === 'ES5') {
             throw new Error(
                 `The "${type}" scenario does not support ES5 output.\n` +
-                `Did you mean to set \`"excludeFromPartialTests": true\` in "${
+                `Did you mean to set \`"compilationModeFilter": ["full compile"]\` in "${
                     test.relativePath}"?`);
           }