refactor(ngcc): abstract task selection behind an interface (#32427)

This change does not alter the current behavior, but makes it easier to
introduce `TaskQueue`s implementing different task selection algorithms,
for example to support executing multiple tasks in parallel (while
respecting interdependencies between them).

Inspired by/Based on @alxhub's prototype: alxhub/angular@cb631bdb1

PR Close #32427
This commit is contained in:
George Kalpakas 2019-08-27 17:36:25 +03:00 committed by Matias Niemelä
parent 0cf94e3ed5
commit 2844dd2972
9 changed files with 468 additions and 13 deletions

View File

@ -9,7 +9,8 @@
import {DepGraph} from 'dependency-graph';
import {AbsoluteFsPath, FileSystem, resolve} from '../../../src/ngtsc/file_system';
import {Logger} from '../logging/logger';
import {EntryPoint, EntryPointFormat, EntryPointJsonProperty, SUPPORTED_FORMAT_PROPERTIES, getEntryPointFormat} from '../packages/entry_point';
import {EntryPoint, EntryPointFormat, SUPPORTED_FORMAT_PROPERTIES, getEntryPointFormat} from '../packages/entry_point';
import {PartiallyOrderedList} from '../utils';
import {DependencyHost, DependencyInfo} from './dependency_host';
const builtinNodeJsModules = new Set<string>(require('module').builtinModules);
@ -51,6 +52,16 @@ export interface DependencyDiagnostics {
ignoredDependencies: IgnoredDependency[];
}
/**
* Represents a partially ordered list of entry-points.
*
* The entry-points' order/precedence is such that dependent entry-points always come later than
* their dependencies in the list.
*
* See `DependencyResolver#sortEntryPointsByDependency()`.
*/
export type PartiallyOrderedEntryPoints = PartiallyOrderedList<EntryPoint>;
/**
* A list of entry-points, sorted by their dependencies.
*
@ -60,7 +71,9 @@ export interface DependencyDiagnostics {
* Some entry points or their dependencies may be have been ignored. These are captured for
* diagnostic purposes in `invalidEntryPoints` and `ignoredDependencies` respectively.
*/
export interface SortedEntryPointsInfo extends DependencyDiagnostics { entryPoints: EntryPoint[]; }
export interface SortedEntryPointsInfo extends DependencyDiagnostics {
entryPoints: PartiallyOrderedEntryPoints;
}
/**
* A class that resolves dependencies between entry-points.
@ -94,7 +107,8 @@ export class DependencyResolver {
}
return {
entryPoints: sortedEntryPointNodes.map(path => graph.getNodeData(path)),
entryPoints: (sortedEntryPointNodes as PartiallyOrderedList<string>)
.map(path => graph.getNodeData(path)),
invalidEntryPoints,
ignoredDependencies,
};

View File

@ -7,6 +7,7 @@
*/
import {EntryPoint, EntryPointJsonProperty} from '../packages/entry_point';
import {PartiallyOrderedList} from '../utils';
/**
@ -15,7 +16,7 @@ import {EntryPoint, EntryPointJsonProperty} from '../packages/entry_point';
* @return A list of tasks that need to be executed in order to process the necessary format
* properties for all entry-points.
*/
export type AnalyzeEntryPointsFn = () => Task[];
export type AnalyzeEntryPointsFn = () => TaskQueue;
/** The type of the function that can process/compile a task. */
export type CompileFn = (task: Task) => void;
@ -32,6 +33,21 @@ export interface Executor {
void|Promise<void>;
}
/**
* Represents a partially ordered list of tasks.
*
* The ordering/precedence of tasks is determined by the inter-dependencies between their associated
* entry-points. Specifically, the tasks' order/precedence is such that tasks associated to
* dependent entry-points always come after tasks associated with their dependencies.
*
* As result of this ordering, it is guaranteed that - by processing tasks in the order in which
* they appear in the list - a task's dependencies will always have been processed before processing
* the task itself.
*
* See `DependencyResolver#sortEntryPointsByDependency()`.
*/
export type PartiallyOrderedTasks = PartiallyOrderedList<Task>;
/** Represents a unit of work: processing a specific format property of an entry-point. */
export interface Task {
/** The `EntryPoint` which needs to be processed as part of the task. */
@ -65,3 +81,43 @@ export const enum TaskProcessingOutcome {
/** Successfully processed the target format property. */
Processed,
}
/**
* A wrapper around a list of tasks and providing utility methods for getting the next task of
* interest and determining when all tasks have been completed.
*
* (This allows different implementations to impose different constraints on when a task's
* processing can start.)
*/
export interface TaskQueue {
/** Whether all tasks have been completed. */
allTasksCompleted: boolean;
/**
* Get the next task whose processing can start (if any).
*
* This implicitly marks the task as in-progress.
* (This information is used to determine whether all tasks have been completed.)
*
* @return The next task available for processing or `null`, if no task can be processed at the
* moment (including if there are no more unprocessed tasks).
*/
getNextTask(): Task|null;
/**
* Mark a task as completed.
*
* This removes the task from the internal list of in-progress tasks.
* (This information is used to determine whether all tasks have been completed.)
*
* @param task The task to mark as completed.
*/
markTaskCompleted(task: Task): void;
/**
* Return a string representation of the task queue (for debugging purposes).
*
* @return A string representation of the task queue.
*/
toString(): string;
}

View File

@ -22,13 +22,15 @@ export class SingleProcessExecutor implements Executor {
execute(analyzeEntryPoints: AnalyzeEntryPointsFn, createCompileFn: CreateCompileFn): void {
this.logger.debug(`Running ngcc on ${this.constructor.name}.`);
const tasks = analyzeEntryPoints();
const taskQueue = analyzeEntryPoints();
const compile =
createCompileFn((task, outcome) => onTaskCompleted(this.pkgJsonUpdater, task, outcome));
// Process all tasks.
for (const task of tasks) {
while (!taskQueue.allTasksCompleted) {
const task = taskQueue.getNextTask() !;
compile(task);
taskQueue.markTaskCompleted(task);
}
}
}

View File

@ -0,0 +1,47 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {PartiallyOrderedTasks, Task, TaskQueue} from '../api';
import {stringifyTask} from '../utils';
/**
* A base `TaskQueue` implementation to be used as base for concrete implementations.
*/
export abstract class BaseTaskQueue implements TaskQueue {
get allTasksCompleted(): boolean {
return (this.tasks.length === 0) && (this.inProgressTasks.size === 0);
}
protected inProgressTasks = new Set<Task>();
constructor(protected tasks: PartiallyOrderedTasks) {}
abstract getNextTask(): Task|null;
markTaskCompleted(task: Task): void {
if (!this.inProgressTasks.has(task)) {
throw new Error(
`Trying to mark task that was not in progress as completed: ${stringifyTask(task)}`);
}
this.inProgressTasks.delete(task);
}
toString(): string {
const inProgTasks = Array.from(this.inProgressTasks);
return `${this.constructor.name}\n` +
` All tasks completed: ${this.allTasksCompleted}\n` +
` Unprocessed tasks (${this.tasks.length}): ${this.stringifyTasks(this.tasks, ' ')}\n` +
` In-progress tasks (${inProgTasks.length}): ${this.stringifyTasks(inProgTasks, ' ')}`;
}
protected stringifyTasks(tasks: Task[], indentation: string): string {
return tasks.map(task => `\n${indentation}- ${stringifyTask(task)}`).join('');
}
}

View File

@ -0,0 +1,37 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {Task} from '../api';
import {stringifyTask} from '../utils';
import {BaseTaskQueue} from './base_task_queue';
/**
* A `TaskQueue` implementation that assumes tasks are processed serially and each one is completed
* before requesting the next one.
*/
export class SerialTaskQueue extends BaseTaskQueue {
getNextTask(): Task|null {
const nextTask = this.tasks.shift() || null;
if (nextTask) {
if (this.inProgressTasks.size > 0) {
// `SerialTaskQueue` can have max one in-progress task.
const inProgressTask = this.inProgressTasks.values().next().value;
throw new Error(
'Trying to get next task, while there is already a task in progress: ' +
stringifyTask(inProgressTask));
}
this.inProgressTasks.add(nextTask);
}
return nextTask;
}
}

View File

@ -32,3 +32,7 @@ export const onTaskCompleted =
pkgJsonUpdater, entryPoint.packageJson, packageJsonPath, propsToMarkAsProcessed);
}
};
/** Stringify a task for debugging purposes. */
export const stringifyTask = (task: Task): string =>
`{entryPoint: ${task.entryPoint.name}, formatProperty: ${task.formatProperty}, processDts: ${task.processDts}}`;

View File

@ -10,19 +10,20 @@ import * as ts from 'typescript';
import {AbsoluteFsPath, FileSystem, absoluteFrom, dirname, getFileSystem, resolve} from '../../src/ngtsc/file_system';
import {CommonJsDependencyHost} from './dependencies/commonjs_dependency_host';
import {DependencyResolver, InvalidEntryPoint, SortedEntryPointsInfo} from './dependencies/dependency_resolver';
import {DependencyResolver, InvalidEntryPoint, PartiallyOrderedEntryPoints, SortedEntryPointsInfo} from './dependencies/dependency_resolver';
import {EsmDependencyHost} from './dependencies/esm_dependency_host';
import {ModuleResolver} from './dependencies/module_resolver';
import {UmdDependencyHost} from './dependencies/umd_dependency_host';
import {DirectoryWalkerEntryPointFinder} from './entry_point_finder/directory_walker_entry_point_finder';
import {TargetedEntryPointFinder} from './entry_point_finder/targeted_entry_point_finder';
import {AnalyzeEntryPointsFn, CreateCompileFn, Executor, Task, TaskProcessingOutcome} from './execution/api';
import {AnalyzeEntryPointsFn, CreateCompileFn, Executor, PartiallyOrderedTasks, Task, TaskProcessingOutcome} from './execution/api';
import {AsyncSingleProcessExecutor, SingleProcessExecutor} from './execution/single_process_executor';
import {SerialTaskQueue} from './execution/task_selection/serial_task_queue';
import {ConsoleLogger, LogLevel} from './logging/console_logger';
import {Logger} from './logging/logger';
import {hasBeenProcessed, markAsProcessed} from './packages/build_marker';
import {NgccConfiguration} from './packages/configuration';
import {EntryPoint, EntryPointJsonProperty, EntryPointPackageJson, SUPPORTED_FORMAT_PROPERTIES, getEntryPointFormat} from './packages/entry_point';
import {EntryPointJsonProperty, EntryPointPackageJson, SUPPORTED_FORMAT_PROPERTIES, getEntryPointFormat} from './packages/entry_point';
import {makeEntryPointBundle} from './packages/entry_point_bundle';
import {Transformer} from './packages/transformer';
import {PathMappings} from './utils';
@ -139,7 +140,8 @@ export function mainNgcc(
targetEntryPointPath, pathMappings, supportedPropertiesToConsider, compileAllFormats);
const unprocessableEntryPointPaths: string[] = [];
const tasks: Task[] = [];
// The tasks are partially ordered by virtue of the entry-points being partially ordered too.
const tasks: PartiallyOrderedTasks = [] as any;
for (const entryPoint of entryPoints) {
const packageJson = entryPoint.packageJson;
@ -174,7 +176,7 @@ export function mainNgcc(
unprocessableEntryPointPaths.map(path => `\n - ${path}`).join(''));
}
return tasks;
return new SerialTaskQueue(tasks);
};
// The function for creating the `compile()` function.
@ -277,7 +279,7 @@ function getEntryPoints(
fs: FileSystem, pkgJsonUpdater: PackageJsonUpdater, logger: Logger,
resolver: DependencyResolver, config: NgccConfiguration, basePath: AbsoluteFsPath,
targetEntryPointPath: string | undefined, pathMappings: PathMappings | undefined,
propertiesToConsider: string[], compileAllFormats: boolean): EntryPoint[] {
propertiesToConsider: string[], compileAllFormats: boolean): PartiallyOrderedEntryPoints {
const {entryPoints, invalidEntryPoints} = (targetEntryPointPath !== undefined) ?
getTargetedEntryPoints(
fs, pkgJsonUpdater, logger, resolver, config, basePath, targetEntryPointPath,
@ -296,7 +298,11 @@ function getTargetedEntryPoints(
if (hasProcessedTargetEntryPoint(
fs, absoluteTargetEntryPointPath, propertiesToConsider, compileAllFormats)) {
logger.debug('The target entry-point has already been processed');
return {entryPoints: [], invalidEntryPoints: [], ignoredDependencies: []};
return {
entryPoints: [] as unknown as PartiallyOrderedEntryPoints,
invalidEntryPoints: [],
ignoredDependencies: [],
};
}
const finder = new TargetedEntryPointFinder(
fs, config, logger, resolver, basePath, absoluteTargetEntryPointPath, pathMappings);

View File

@ -8,6 +8,28 @@
import * as ts from 'typescript';
import {AbsoluteFsPath, FileSystem, absoluteFrom} from '../../src/ngtsc/file_system';
/**
* A list (`Array`) of partially ordered `T` items.
*
* The items in the list are partially ordered in the sense that any element has either the same or
* higher precedence than any element which appears later in the list. What "higher precedence"
* means and how it is determined is implementation-dependent.
*
* See [PartiallyOrderedSet](https://en.wikipedia.org/wiki/Partially_ordered_set) for more details.
* (Refraining from using the term "set" here, to avoid confusion with JavaScript's
* [Set](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Set).)
*
* NOTE: A plain `Array<T>` is not assignable to a `PartiallyOrderedList<T>`, but a
* `PartiallyOrderedList<T>` is assignable to an `Array<T>`.
*/
export interface PartiallyOrderedList<T> extends Array<T> {
_partiallyOrdered: true;
map<U>(callbackfn: (value: T, index: number, array: PartiallyOrderedList<T>) => U, thisArg?: any):
PartiallyOrderedList<U>;
slice(...args: Parameters<Array<T>['slice']>): PartiallyOrderedList<T>;
}
export function getOriginalSymbol(checker: ts.TypeChecker): (symbol: ts.Symbol) => ts.Symbol {
return function(symbol: ts.Symbol) {
return ts.SymbolFlags.Alias & symbol.flags ? checker.getAliasedSymbol(symbol) : symbol;

View File

@ -0,0 +1,267 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {PartiallyOrderedTasks, Task, TaskQueue} from '../../../src/execution/api';
import {SerialTaskQueue} from '../../../src/execution/task_selection/serial_task_queue';
describe('SerialTaskQueue', () => {
// Helpers
/**
* Create a `TaskQueue` by generating mock tasks.
*
* NOTE: Tasks at even indices generate typings.
*
* @param taskCount The number of tasks to generate.
* @return An object with the following properties:
* - `tasks`: The (partially ordered) list of generated mock tasks.
* - `queue`: The created `TaskQueue`.
*/
const createQueue = (taskCount: number): {tasks: PartiallyOrderedTasks, queue: TaskQueue} => {
const tasks: PartiallyOrderedTasks = [] as any;
for (let i = 0; i < taskCount; i++) {
tasks.push({
entryPoint: {name: `entry-point-${i}`}, formatProperty: `prop-${i}`,
processDts: i % 2 === 0,
} as Task);
}
return {tasks, queue: new SerialTaskQueue(tasks.slice())};
};
/**
* Simulate processing the next task:
* - Request the next task from the specified queue.
* - If a task was returned, mark it as completed.
* - Return the task (this allows making assertions against the picked tasks in tests).
*
* @param queue The `TaskQueue` to get the next task from.
* @return The "processed" task (if any).
*/
const processNextTask = (queue: TaskQueue): ReturnType<TaskQueue['getNextTask']> => {
const task = queue.getNextTask();
if (task !== null) queue.markTaskCompleted(task);
return task;
};
describe('allTasksCompleted', () => {
it('should be `false`, when there are unprocessed tasks', () => {
const {queue} = createQueue(2);
expect(queue.allTasksCompleted).toBe(false);
processNextTask(queue);
expect(queue.allTasksCompleted).toBe(false);
});
it('should be `false`, when there are tasks in progress', () => {
const {queue} = createQueue(1);
queue.getNextTask();
expect(queue.allTasksCompleted).toBe(false);
});
it('should be `true`, when there are no unprocessed or in-progress tasks', () => {
const {queue} = createQueue(3);
processNextTask(queue);
expect(queue.allTasksCompleted).toBe(false);
processNextTask(queue);
expect(queue.allTasksCompleted).toBe(false);
processNextTask(queue);
expect(queue.allTasksCompleted).toBe(true);
});
it('should be `true`, if the queue was empty from the beginning', () => {
const {queue} = createQueue(0);
expect(queue.allTasksCompleted).toBe(true);
});
it('should remain `true` once the queue has been emptied', () => {
const {queue} = createQueue(1);
expect(queue.allTasksCompleted).toBe(false);
processNextTask(queue);
expect(queue.allTasksCompleted).toBe(true);
processNextTask(queue);
expect(queue.allTasksCompleted).toBe(true);
});
});
describe('getNextTask()', () => {
it('should return the tasks in order', () => {
const {tasks, queue} = createQueue(3);
expect(processNextTask(queue)).toBe(tasks[0]);
expect(processNextTask(queue)).toBe(tasks[1]);
expect(processNextTask(queue)).toBe(tasks[2]);
});
it('should return `null`, when there are no more tasks', () => {
const {tasks, queue} = createQueue(3);
tasks.forEach(() => expect(processNextTask(queue)).not.toBe(null));
expect(processNextTask(queue)).toBe(null);
expect(processNextTask(queue)).toBe(null);
const {tasks: tasks2, queue: queue2} = createQueue(0);
expect(tasks2).toEqual([]);
expect(processNextTask(queue2)).toBe(null);
expect(processNextTask(queue2)).toBe(null);
});
it('should throw, if a task is already in progress', () => {
const {queue} = createQueue(3);
queue.getNextTask();
expect(() => queue.getNextTask())
.toThrowError(
`Trying to get next task, while there is already a task in progress: ` +
`{entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}`);
});
});
describe('markTaskCompleted()', () => {
it('should mark a task as completed, so that the next task can be picked', () => {
const {queue} = createQueue(3);
const task = queue.getNextTask() !;
expect(() => queue.getNextTask()).toThrow();
queue.markTaskCompleted(task);
expect(() => queue.getNextTask()).not.toThrow();
});
it('should throw, if the specified task is not in progress', () => {
const {tasks, queue} = createQueue(3);
queue.getNextTask();
expect(() => queue.markTaskCompleted(tasks[2]))
.toThrowError(
`Trying to mark task that was not in progress as completed: ` +
`{entryPoint: entry-point-2, formatProperty: prop-2, processDts: true}`);
});
});
describe('toString()', () => {
it('should include the `TaskQueue` constructor\'s name', () => {
const {queue} = createQueue(0);
expect(queue.toString()).toMatch(/^SerialTaskQueue\n/);
});
it('should include the value of `allTasksCompleted`', () => {
const {queue: queue1} = createQueue(0);
expect(queue1.toString()).toContain(' All tasks completed: true\n');
const {queue: queue2} = createQueue(3);
expect(queue2.toString()).toContain(' All tasks completed: false\n');
processNextTask(queue2);
processNextTask(queue2);
const task = queue2.getNextTask() !;
expect(queue2.toString()).toContain(' All tasks completed: false\n');
queue2.markTaskCompleted(task);
expect(queue2.toString()).toContain(' All tasks completed: true\n');
});
it('should include the unprocessed tasks', () => {
const {queue} = createQueue(3);
expect(queue.toString())
.toContain(
' Unprocessed tasks (3): \n' +
' - {entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}\n' +
' - {entryPoint: entry-point-1, formatProperty: prop-1, processDts: false}\n' +
' - {entryPoint: entry-point-2, formatProperty: prop-2, processDts: true}\n');
const task1 = queue.getNextTask() !;
expect(queue.toString())
.toContain(
' Unprocessed tasks (2): \n' +
' - {entryPoint: entry-point-1, formatProperty: prop-1, processDts: false}\n' +
' - {entryPoint: entry-point-2, formatProperty: prop-2, processDts: true}\n');
queue.markTaskCompleted(task1);
const task2 = queue.getNextTask() !;
expect(queue.toString())
.toContain(
' Unprocessed tasks (1): \n' +
' - {entryPoint: entry-point-2, formatProperty: prop-2, processDts: true}\n');
queue.markTaskCompleted(task2);
processNextTask(queue);
expect(queue.toString()).toContain(' Unprocessed tasks (0): \n');
});
it('should include the in-progress tasks', () => {
const {queue} = createQueue(3);
expect(queue.toString()).toContain(' In-progress tasks (0): ');
const task1 = queue.getNextTask() !;
expect(queue.toString())
.toContain(
' In-progress tasks (1): \n' +
' - {entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}');
queue.markTaskCompleted(task1);
const task2 = queue.getNextTask() !;
expect(queue.toString())
.toContain(
' In-progress tasks (1): \n' +
' - {entryPoint: entry-point-1, formatProperty: prop-1, processDts: false}');
queue.markTaskCompleted(task2);
processNextTask(queue);
expect(queue.toString()).toContain(' In-progress tasks (0): ');
});
it('should display all info together', () => {
const {queue: queue1} = createQueue(0);
expect(queue1.toString())
.toBe(
'SerialTaskQueue\n' +
' All tasks completed: true\n' +
' Unprocessed tasks (0): \n' +
' In-progress tasks (0): ');
const {queue: queue2} = createQueue(3);
expect(queue2.toString())
.toBe(
'SerialTaskQueue\n' +
' All tasks completed: false\n' +
' Unprocessed tasks (3): \n' +
' - {entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}\n' +
' - {entryPoint: entry-point-1, formatProperty: prop-1, processDts: false}\n' +
' - {entryPoint: entry-point-2, formatProperty: prop-2, processDts: true}\n' +
' In-progress tasks (0): ');
processNextTask(queue2);
const task = queue2.getNextTask() !;
expect(queue2.toString())
.toBe(
'SerialTaskQueue\n' +
' All tasks completed: false\n' +
' Unprocessed tasks (1): \n' +
' - {entryPoint: entry-point-2, formatProperty: prop-2, processDts: true}\n' +
' In-progress tasks (1): \n' +
' - {entryPoint: entry-point-1, formatProperty: prop-1, processDts: false}');
queue2.markTaskCompleted(task);
processNextTask(queue2);
expect(queue2.toString())
.toBe(
'SerialTaskQueue\n' +
' All tasks completed: true\n' +
' Unprocessed tasks (0): \n' +
' In-progress tasks (0): ');
});
});
});