refactor(ivy): ngcc - split work into distinct analyze/compile/execute phases (#32052)

This refactoring more clearly separates the different phases of the work
performed by `ngcc`, setting the ground for being able to run each phase
independently in the future and improve performance via parallelization.

Inspired by/Based on @alxhub's prototype: alxhub/angular@cb631bdb1

PR Close #32052

packages/compiler-cli/ngcc/src/execution/api.ts

@@ -0,0 +1,74 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {EntryPoint, EntryPointJsonProperty} from '../packages/entry_point';
+
+/** The type of the function that analyzes entry-points and creates the list of tasks. */
+export type AnalyzeFn = () => {
+  processingMetadataPerEntryPoint: Map<string, EntryPointProcessingMetadata>;
+  tasks: Task[];
+};
+
+/**
+ * The type of the function that creates the `compile()` function, which in turn can be used to
+ * process tasks.
+ */
+export type CreateCompileFn =
+    (onTaskCompleted: (task: Task, outcome: TaskProcessingOutcome) => void) => (task: Task) => void;
+
+/**
+ * The type of the function that orchestrates and executes the required work (i.e. analyzes the
+ * entry-points, processes the resulting tasks, does book-keeping and validates the final outcome).
+ */
+export type ExecuteFn = (analyzeFn: AnalyzeFn, createCompileFn: CreateCompileFn) => void;
+
+/** Represents metadata related to the processing of an entry-point. */
+export interface EntryPointProcessingMetadata {
+  /**
+   * A mapping from a format property (i.e. an `EntryPointJsonProperty`) to the list of format
+   * properties that point to the same format-path and as a result need to be marked as processed,
+   * once the former is processed.
+   */
+  propertyToPropertiesToMarkAsProcessed: Map<EntryPointJsonProperty, EntryPointJsonProperty[]>;
+
+  /**
+   * Whether the typings for the entry-point have been successfully processed (or were already
+   * processed).
+   */
+  hasProcessedTypings: boolean;
+
+  /**
+   * Whether at least one format has been successfully processed (or was already processed) for the
+   * entry-point.
+   */
+  hasAnyProcessedFormat: boolean;
+}
+
+/** Represents a unit of work: processing a specific format property of an entry-point. */
+export interface Task {
+  /** The `EntryPoint` which needs to be processed as part of the task. */
+  entryPoint: EntryPoint;
+
+  /**
+   * The `package.json` format property to process (i.e. the property which points to the file that
+   * is the program entry-point).
+   */
+  formatProperty: EntryPointJsonProperty;
+
+  /** Whether to also process typings for this entry-point as part of the task. */
+  processDts: boolean;
+}
+
+/** Represents the outcome of processing a `Task`. */
+export const enum TaskProcessingOutcome {
+  /** The target format property was already processed - didn't have to do anything. */
+  AlreadyProcessed,
+
+  /** Successfully processed the target format property. */
+  Processed,
+}

packages/compiler-cli/ngcc/src/main.ts

@@ -14,6 +14,7 @@ import {ModuleResolver} from './dependencies/module_resolver';
 import {UmdDependencyHost} from './dependencies/umd_dependency_host';
 import {DirectoryWalkerEntryPointFinder} from './entry_point_finder/directory_walker_entry_point_finder';
 import {TargetedEntryPointFinder} from './entry_point_finder/targeted_entry_point_finder';
+import {AnalyzeFn, CreateCompileFn, EntryPointProcessingMetadata, ExecuteFn, Task, TaskProcessingOutcome} from './execution/api';
 import {ConsoleLogger, LogLevel} from './logging/console_logger';
 import {Logger} from './logging/logger';
 import {hasBeenProcessed, markAsProcessed} from './packages/build_marker';
@@ -82,46 +83,68 @@ export function mainNgcc(
     {basePath, targetEntryPointPath, propertiesToConsider = SUPPORTED_FORMAT_PROPERTIES,
      compileAllFormats = true, createNewEntryPointFormats = false,
      logger = new ConsoleLogger(LogLevel.info), pathMappings}: NgccOptions): void {
-  const supportedPropertiesToConsider = ensureSupportedProperties(propertiesToConsider);
-
   const fileSystem = getFileSystem();
-  const transformer = new Transformer(fileSystem, logger);
-  const moduleResolver = new ModuleResolver(fileSystem, pathMappings);
-  const esmDependencyHost = new EsmDependencyHost(fileSystem, moduleResolver);
-  const umdDependencyHost = new UmdDependencyHost(fileSystem, moduleResolver);
-  const commonJsDependencyHost = new CommonJsDependencyHost(fileSystem, moduleResolver);
-  const resolver = new DependencyResolver(fileSystem, logger, {
-    esm5: esmDependencyHost,
-    esm2015: esmDependencyHost,
-    umd: umdDependencyHost,
-    commonjs: commonJsDependencyHost
-  });
-  const absBasePath = absoluteFrom(basePath);
-  const config = new NgccConfiguration(fileSystem, dirname(absBasePath));
-  const fileWriter = getFileWriter(fileSystem, createNewEntryPointFormats);
-  const entryPoints = getEntryPoints(
-      fileSystem, config, logger, resolver, absBasePath, targetEntryPointPath, pathMappings,
-      supportedPropertiesToConsider, compileAllFormats);
 
-  for (const entryPoint of entryPoints) {
+  // The function for performing the analysis.
-    // Are we compiling the Angular core?
+  const analyzeFn: AnalyzeFn = () => {
-    const isCore = entryPoint.name === '@angular/core';
+    const supportedPropertiesToConsider = ensureSupportedProperties(propertiesToConsider);
 
-    const entryPointPackageJson = entryPoint.packageJson;
+    const moduleResolver = new ModuleResolver(fileSystem, pathMappings);
-    const entryPointPackageJsonPath = fileSystem.resolve(entryPoint.path, 'package.json');
+    const esmDependencyHost = new EsmDependencyHost(fileSystem, moduleResolver);
-    const {propertiesToProcess, propertyToPropertiesToMarkAsProcessed} =
+    const umdDependencyHost = new UmdDependencyHost(fileSystem, moduleResolver);
-        getPropertiesToProcessAndMarkAsProcessed(
+    const commonJsDependencyHost = new CommonJsDependencyHost(fileSystem, moduleResolver);
-            entryPointPackageJson, supportedPropertiesToConsider);
+    const dependencyResolver = new DependencyResolver(fileSystem, logger, {
+      esm5: esmDependencyHost,
+      esm2015: esmDependencyHost,
+      umd: umdDependencyHost,
+      commonjs: commonJsDependencyHost
+    });
 
-    let hasAnyProcessedFormat = false;
+    const absBasePath = absoluteFrom(basePath);
-    let processDts = !hasBeenProcessed(entryPointPackageJson, 'typings');
+    const config = new NgccConfiguration(fileSystem, dirname(absBasePath));
+    const entryPoints = getEntryPoints(
+        fileSystem, config, logger, dependencyResolver, absBasePath, targetEntryPointPath,
+        pathMappings, supportedPropertiesToConsider, compileAllFormats);
 
-    for (const property of propertiesToProcess) {
+    const processingMetadataPerEntryPoint = new Map<string, EntryPointProcessingMetadata>();
-      // If we only need one format processed and we already have one, exit the loop.
+    const tasks: Task[] = [];
-      if (!compileAllFormats && hasAnyProcessedFormat) break;
 
-      const formatPath = entryPointPackageJson[property];
+    for (const entryPoint of entryPoints) {
-      const format = getEntryPointFormat(fileSystem, entryPoint, property);
+      const packageJson = entryPoint.packageJson;
+      const hasProcessedTypings = hasBeenProcessed(packageJson, 'typings');
+      const {propertiesToProcess, propertyToPropertiesToMarkAsProcessed} =
+          getPropertiesToProcessAndMarkAsProcessed(packageJson, supportedPropertiesToConsider);
+      let processDts = !hasProcessedTypings;
+
+      for (const formatProperty of propertiesToProcess) {
+        tasks.push({entryPoint, formatProperty, processDts});
+
+        // Only process typings for the first property (if not already processed).
+        processDts = false;
+      }
+
+      processingMetadataPerEntryPoint.set(entryPoint.path, {
+        propertyToPropertiesToMarkAsProcessed,
+        hasProcessedTypings,
+        hasAnyProcessedFormat: false,
+      });
+    }
+
+    return {processingMetadataPerEntryPoint, tasks};
+  };
+
+  // The function for creating the `compile()` function.
+  const createCompileFn: CreateCompileFn = onTaskCompleted => {
+    const fileWriter = getFileWriter(fileSystem, createNewEntryPointFormats);
+    const transformer = new Transformer(fileSystem, logger);
+
+    return (task: Task) => {
+      const {entryPoint, formatProperty, processDts} = task;
+
+      const isCore = entryPoint.name === '@angular/core';  // Are we compiling the Angular core?
+      const packageJson = entryPoint.packageJson;
+      const formatPath = packageJson[formatProperty];
+      const format = getEntryPointFormat(fileSystem, entryPoint, formatProperty);
 
       // All properties listed in `propertiesToProcess` are guaranteed to point to a format-path
       // (i.e. they exist in `entryPointPackageJson`). Furthermore, they are also guaranteed to be
@@ -131,42 +154,78 @@ export function mainNgcc(
       if (!formatPath || !format) {
         // This should never happen.
         throw new Error(
-            `Invariant violated: No format-path or format for ${entryPoint.path} : ${property} ` +
+            `Invariant violated: No format-path or format for ${entryPoint.path} : ` +
-            `(formatPath: ${formatPath} | format: ${format})`);
+            `${formatProperty} (formatPath: ${formatPath} | format: ${format})`);
       }
 
-      // The `formatPath` which the property maps to is already processed - nothing to do.
+      // The format-path which the property maps to is already processed - nothing to do.
-      if (hasBeenProcessed(entryPointPackageJson, property)) {
+      if (hasBeenProcessed(packageJson, formatProperty)) {
-        hasAnyProcessedFormat = true;
+        logger.debug(`Skipping ${entryPoint.name} : ${formatProperty} (already compiled).`);
-        logger.debug(`Skipping ${entryPoint.name} : ${property} (already compiled).`);
+        onTaskCompleted(task, TaskProcessingOutcome.AlreadyProcessed);
-        continue;
+        return;
       }
 
       const bundle = makeEntryPointBundle(
-          fileSystem, entryPoint, formatPath, isCore, property, format, processDts, pathMappings,
+          fileSystem, entryPoint, formatPath, isCore, formatProperty, format, processDts,
-          true);
+          pathMappings, true);
+
+      logger.info(`Compiling ${entryPoint.name} : ${formatProperty} as ${format}`);
 
-      logger.info(`Compiling ${entryPoint.name} : ${property} as ${format}`);
       const transformedFiles = transformer.transform(bundle);
       fileWriter.writeBundle(entryPoint, bundle, transformedFiles);
-      hasAnyProcessedFormat = true;
 
-      const propsToMarkAsProcessed: (EntryPointJsonProperty | 'typings')[] =
+      onTaskCompleted(task, TaskProcessingOutcome.Processed);
-          propertyToPropertiesToMarkAsProcessed.get(property) !;
+    };
-      if (processDts) {
+  };
-        propsToMarkAsProcessed.push('typings');
+
-        processDts = false;
+  // The function for actually planning and getting the work done.
+  const executeFn: ExecuteFn = (analyzeFn: AnalyzeFn, createCompileFn: CreateCompileFn) => {
+    const {processingMetadataPerEntryPoint, tasks} = analyzeFn();
+    const compile = createCompileFn(({entryPoint, formatProperty, processDts}, outcome) => {
+      const processingMeta = processingMetadataPerEntryPoint.get(entryPoint.path) !;
+      processingMeta.hasAnyProcessedFormat = true;
+
+      if (outcome === TaskProcessingOutcome.Processed) {
+        const packageJsonPath = fileSystem.resolve(entryPoint.path, 'package.json');
+        const propsToMarkAsProcessed: (EntryPointJsonProperty | 'typings')[] =
+            processingMeta.propertyToPropertiesToMarkAsProcessed.get(formatProperty) !;
+
+        if (processDts) {
+          processingMeta.hasProcessedTypings = true;
+          propsToMarkAsProcessed.push('typings');
+        }
+
+        markAsProcessed(
+            fileSystem, entryPoint.packageJson, packageJsonPath, propsToMarkAsProcessed);
       }
+    });
 
-      markAsProcessed(
+    // Process all tasks.
-          fileSystem, entryPointPackageJson, entryPointPackageJsonPath, propsToMarkAsProcessed);
+    for (const task of tasks) {
+      const processingMeta = processingMetadataPerEntryPoint.get(task.entryPoint.path) !;
+
+      // If we only need one format processed and we already have one for the corresponding
+      // entry-point, skip the task.
+      if (!compileAllFormats && processingMeta.hasAnyProcessedFormat) continue;
+
+      compile(task);
     }
 
-    if (!hasAnyProcessedFormat) {
+    // Check for entry-points for which we could not process any format at all.
+    const unprocessedEntryPointPaths =
+        Array.from(processingMetadataPerEntryPoint.entries())
+            .filter(([, processingMeta]) => !processingMeta.hasAnyProcessedFormat)
+            .map(([entryPointPath]) => `\n  - ${entryPointPath}`)
+            .join('');
+
+    if (unprocessedEntryPointPaths) {
       throw new Error(
-          `Failed to compile any formats for entry-point at (${entryPoint.path}). Tried ${supportedPropertiesToConsider}.`);
+          'Failed to compile any formats for the following entry-points (tried ' +
+          `${propertiesToConsider.join(', ')}): ${unprocessedEntryPointPaths}`);
     }
-  }
+  };
+
+  return executeFn(analyzeFn, createCompileFn);
 }
 
 function ensureSupportedProperties(properties: string[]): EntryPointJsonProperty[] {