refactor(ngcc): expose the TaskDependencies mapping on BaseTaskQueue (#36083)
Later when we implement the ability to continue processing when tasks have failed to compile, we will also need to avoid processing tasks that depend upon the failed task. This refactoring exposes this list of dependent tasks in a way that can be used to skip processing of tasks that depend upon a failed task. It also changes the blocking model of the parallel mode of operation so that non-typings tasks are now blocked on their corresponding typings task. Previously the non-typings tasks could be triggered to run in parallel to the typings task, since they do not have a hard dependency on each other, but this made it difficult to skip task correctly if the typings task failed, since it was possible that a non-typings task was already in flight when the typings task failed. The result of this is a small potential degradation of performance in async parallel processing mode, in the rare cases that there were not enough unblocked tasks to make use of all the available workers. PR Close #36083
This commit is contained in:
Pete Bacon Darwin
Andrew Kushnir
parent
39d4016fe9
commit
1790b63a5d
|
|
@ -51,6 +51,12 @@ export interface Task extends JsonObject {
|
|||
*/
|
||||
export type PartiallyOrderedTasks = PartiallyOrderedList<Task>;
|
||||
|
||||
/**
|
||||
* A mapping from Tasks to the Tasks that depend upon them (dependents).
|
||||
*/
|
||||
export type TaskDependencies = Map<Task, Set<Task>>;
|
||||
export const TaskDependencies = Map;
|
||||
|
||||
/**
|
||||
* A function to create a TaskCompletedCallback function.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {PartiallyOrderedTasks, Task, TaskQueue} from '../api';
|
||||
import {PartiallyOrderedTasks, Task, TaskDependencies, TaskQueue} from '../api';
|
||||
import {stringifyTask} from '../utils';
|
||||
|
||||
|
||||
|
|
@ -19,7 +19,7 @@ export abstract class BaseTaskQueue implements TaskQueue {
|
|||
}
|
||||
protected inProgressTasks = new Set<Task>();
|
||||
|
||||
constructor(protected tasks: PartiallyOrderedTasks) {}
|
||||
constructor(protected tasks: PartiallyOrderedTasks, protected dependencies: TaskDependencies) {}
|
||||
|
||||
abstract getNextTask(): Task|null;
|
||||
|
||||
|
|
|
|||
|
|
@ -5,39 +5,25 @@
|
|||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {DepGraph} from 'dependency-graph';
|
||||
|
||||
import {EntryPoint} from '../../../packages/entry_point';
|
||||
import {PartiallyOrderedTasks, Task} from '../api';
|
||||
import {stringifyTask} from '../utils';
|
||||
|
||||
import {PartiallyOrderedTasks, Task, TaskDependencies} from '../api';
|
||||
import {getBlockedTasks, sortTasksByPriority, stringifyTask} from '../utils';
|
||||
import {BaseTaskQueue} from './base_task_queue';
|
||||
|
||||
|
||||
/**
|
||||
* A `TaskQueue` implementation that assumes tasks are processed in parallel, thus has to ensure a
|
||||
* task's dependencies have been processed before processing the task.
|
||||
*/
|
||||
export class ParallelTaskQueue extends BaseTaskQueue {
|
||||
/**
|
||||
* A mapping from each task to the list of tasks that are blocking it (if any).
|
||||
* A map from Tasks to the Tasks that it depends upon.
|
||||
*
|
||||
* A task can block another task, if the latter's entry-point depends on the former's entry-point
|
||||
* _and_ the former is also generating typings (i.e. has `processDts: true`).
|
||||
*
|
||||
* NOTE: If a task is not generating typings, then it cannot affect anything which depends on its
|
||||
* entry-point, regardless of the dependency graph. To put this another way, only the task
|
||||
* which produces the typings for a dependency needs to have been completed.
|
||||
* This is the reverse mapping of `TaskDependencies`.
|
||||
*/
|
||||
private blockedTasks: Map<Task, Set<Task>>;
|
||||
|
||||
constructor(tasks: PartiallyOrderedTasks, graph: DepGraph<EntryPoint>) {
|
||||
const blockedTasks = computeBlockedTasks(tasks, graph);
|
||||
const sortedTasks = sortTasksByPriority(tasks, blockedTasks);
|
||||
|
||||
super(sortedTasks);
|
||||
this.blockedTasks = blockedTasks;
|
||||
constructor(tasks: PartiallyOrderedTasks, dependents: TaskDependencies) {
|
||||
super(sortTasksByPriority(tasks, dependents), dependents);
|
||||
this.blockedTasks = getBlockedTasks(dependents);
|
||||
}
|
||||
|
||||
getNextTask(): Task|null {
|
||||
|
|
@ -57,22 +43,22 @@ export class ParallelTaskQueue extends BaseTaskQueue {
|
|||
markTaskCompleted(task: Task): void {
|
||||
super.markTaskCompleted(task);
|
||||
|
||||
const unblockedTasks: Task[] = [];
|
||||
if (!this.dependencies.has(task)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Remove the completed task from the lists of tasks blocking other tasks.
|
||||
for (const [otherTask, blockingTasks] of Array.from(this.blockedTasks)) {
|
||||
if (blockingTasks.has(task)) {
|
||||
// Unblock the tasks that are dependent upon `task`
|
||||
for (const dependentTask of this.dependencies.get(task) !) {
|
||||
if (this.blockedTasks.has(dependentTask)) {
|
||||
const blockingTasks = this.blockedTasks.get(dependentTask) !;
|
||||
// Remove the completed task from the lists of tasks blocking other tasks.
|
||||
blockingTasks.delete(task);
|
||||
|
||||
// If the other task is not blocked any more, mark it for unblocking.
|
||||
if (blockingTasks.size === 0) {
|
||||
unblockedTasks.push(otherTask);
|
||||
// If the dependent task is not blocked any more, mark it for unblocking.
|
||||
this.blockedTasks.delete(dependentTask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Unblock tasks that are no longer blocked.
|
||||
unblockedTasks.forEach(task => this.blockedTasks.delete(task));
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
|
|
@ -89,88 +75,3 @@ export class ParallelTaskQueue extends BaseTaskQueue {
|
|||
.join('');
|
||||
}
|
||||
}
|
||||
|
||||
// Helpers
|
||||
|
||||
/**
|
||||
* Compute a mapping of blocked tasks to the tasks that are blocking them.
|
||||
*
|
||||
* As a performance optimization, we take into account the fact that `tasks` are sorted in such a
|
||||
* way that a task can only be blocked by earlier tasks (i.e. dependencies always come before
|
||||
* dependants in the list of tasks).
|
||||
*
|
||||
* @param tasks A (partially ordered) list of tasks.
|
||||
* @param graph The dependency graph between entry-points.
|
||||
* @return The map of blocked tasks to the tasks that are blocking them.
|
||||
*/
|
||||
function computeBlockedTasks(
|
||||
tasks: PartiallyOrderedTasks, graph: DepGraph<EntryPoint>): Map<Task, Set<Task>> {
|
||||
const blockedTasksMap = new Map<Task, Set<Task>>();
|
||||
const candidateBlockers = new Map<string, Task>();
|
||||
|
||||
tasks.forEach(task => {
|
||||
// Find the earlier tasks (`candidateBlockers`) that are blocking this task.
|
||||
const deps = graph.dependenciesOf(task.entryPoint.path);
|
||||
const blockingTasks =
|
||||
deps.filter(dep => candidateBlockers.has(dep)).map(dep => candidateBlockers.get(dep) !);
|
||||
|
||||
// If this task is blocked, add it to the map of blocked tasks.
|
||||
if (blockingTasks.length > 0) {
|
||||
blockedTasksMap.set(task, new Set(blockingTasks));
|
||||
}
|
||||
|
||||
// If this task can be potentially blocking (i.e. it generates typings), add it to the list
|
||||
// of candidate blockers for subsequent tasks.
|
||||
if (task.processDts) {
|
||||
const entryPointPath = task.entryPoint.path;
|
||||
|
||||
// There should only be one task per entry-point that generates typings (and thus can block
|
||||
// other tasks), so the following should theoretically never happen, but check just in case.
|
||||
if (candidateBlockers.has(entryPointPath)) {
|
||||
const otherTask = candidateBlockers.get(entryPointPath) !;
|
||||
|
||||
throw new Error(
|
||||
'Invariant violated: Multiple tasks are assigned generating typings for ' +
|
||||
`'${entryPointPath}':\n - ${stringifyTask(otherTask)}\n - ${stringifyTask(task)}`);
|
||||
}
|
||||
|
||||
candidateBlockers.set(entryPointPath, task);
|
||||
}
|
||||
});
|
||||
|
||||
return blockedTasksMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort a list of tasks by priority.
|
||||
*
|
||||
* Priority is determined by the number of other tasks that a task is (transitively) blocking:
|
||||
* The more tasks a task is blocking the higher its priority is, because processing it will
|
||||
* potentially unblock more tasks.
|
||||
*
|
||||
* To keep the behavior predictable, if two tasks block the same number of other tasks, their
|
||||
* relative order in the original `tasks` lists is preserved.
|
||||
*
|
||||
* @param tasks A (partially ordered) list of tasks.
|
||||
* @param blockedTasks A mapping from a task to the list of tasks that are blocking it (if any).
|
||||
* @return The list of tasks sorted by priority.
|
||||
*/
|
||||
function sortTasksByPriority(
|
||||
tasks: PartiallyOrderedTasks, blockedTasks: Map<Task, Set<Task>>): PartiallyOrderedTasks {
|
||||
const priorityPerTask = new Map<Task, [number, number]>();
|
||||
const allBlockingTaskSets = Array.from(blockedTasks.values());
|
||||
const computePriority = (task: Task, idx: number): [number, number] =>
|
||||
[allBlockingTaskSets.reduce(
|
||||
(count, blockingTasks) => count + (blockingTasks.has(task) ? 1 : 0), 0),
|
||||
idx,
|
||||
];
|
||||
|
||||
tasks.forEach((task, i) => priorityPerTask.set(task, computePriority(task, i)));
|
||||
|
||||
return tasks.slice().sort((task1, task2) => {
|
||||
const [p1, idx1] = priorityPerTask.get(task1) !;
|
||||
const [p2, idx2] = priorityPerTask.get(task2) !;
|
||||
|
||||
return (p2 - p1) || (idx1 - idx2);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,8 +5,134 @@
|
|||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
import {Task} from './api';
|
||||
import {DepGraph} from 'dependency-graph';
|
||||
import {EntryPoint} from '../../packages/entry_point';
|
||||
import {PartiallyOrderedTasks, Task, TaskDependencies} from './api';
|
||||
|
||||
/** Stringify a task for debugging purposes. */
|
||||
export const stringifyTask = (task: Task): string =>
|
||||
`{entryPoint: ${task.entryPoint.name}, formatProperty: ${task.formatProperty}, processDts: ${task.processDts}}`;
|
||||
|
||||
/**
|
||||
* Compute a mapping of tasks to the tasks that are dependent on them (if any).
|
||||
*
|
||||
* Task A can depend upon task B, if either:
|
||||
*
|
||||
* * A and B have the same entry-point _and_ B is generating the typings for that entry-point
|
||||
* (i.e. has `processDts: true`).
|
||||
* * A's entry-point depends on B's entry-point _and_ B is also generating typings.
|
||||
*
|
||||
* NOTE: If a task is not generating typings, then it cannot affect anything which depends on its
|
||||
* entry-point, regardless of the dependency graph. To put this another way, only the task
|
||||
* which produces the typings for a dependency needs to have been completed.
|
||||
*
|
||||
* As a performance optimization, we take into account the fact that `tasks` are sorted in such a
|
||||
* way that a task can only depend on earlier tasks (i.e. dependencies always come before
|
||||
* dependents in the list of tasks).
|
||||
*
|
||||
* @param tasks A (partially ordered) list of tasks.
|
||||
* @param graph The dependency graph between entry-points.
|
||||
* @return A map from each task to those tasks directly dependent upon it.
|
||||
*/
|
||||
export function computeTaskDependencies(
|
||||
tasks: PartiallyOrderedTasks, graph: DepGraph<EntryPoint>): TaskDependencies {
|
||||
const dependencies = new TaskDependencies();
|
||||
const candidateDependencies = new Map<string, Task>();
|
||||
|
||||
tasks.forEach(task => {
|
||||
const entryPointPath = task.entryPoint.path;
|
||||
|
||||
// Find the earlier tasks (`candidateDependencies`) that this task depends upon.
|
||||
const deps = graph.dependenciesOf(entryPointPath);
|
||||
const taskDependencies = deps.filter(dep => candidateDependencies.has(dep))
|
||||
.map(dep => candidateDependencies.get(dep) !);
|
||||
|
||||
// If this task has dependencies, add it to the dependencies and dependents maps.
|
||||
if (taskDependencies.length > 0) {
|
||||
for (const dependency of taskDependencies) {
|
||||
const taskDependents = getDependentsSet(dependencies, dependency);
|
||||
taskDependents.add(task);
|
||||
}
|
||||
}
|
||||
|
||||
if (task.processDts) {
|
||||
// SANITY CHECK:
|
||||
// There should only be one task per entry-point that generates typings (and thus can be a
|
||||
// dependency of other tasks), so the following should theoretically never happen, but check
|
||||
// just in case.
|
||||
if (candidateDependencies.has(entryPointPath)) {
|
||||
const otherTask = candidateDependencies.get(entryPointPath) !;
|
||||
throw new Error(
|
||||
'Invariant violated: Multiple tasks are assigned generating typings for ' +
|
||||
`'${entryPointPath}':\n - ${stringifyTask(otherTask)}\n - ${stringifyTask(task)}`);
|
||||
}
|
||||
// This task can potentially be a dependency (i.e. it generates typings), so add it to the
|
||||
// list of candidate dependencies for subsequent tasks.
|
||||
candidateDependencies.set(entryPointPath, task);
|
||||
} else {
|
||||
// This task is not generating typings so we need to add it to the dependents of the task that
|
||||
// does generate typings, if that exists
|
||||
if (candidateDependencies.has(entryPointPath)) {
|
||||
const typingsTask = candidateDependencies.get(entryPointPath) !;
|
||||
const typingsTaskDependents = getDependentsSet(dependencies, typingsTask);
|
||||
typingsTaskDependents.add(task);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return dependencies;
|
||||
}
|
||||
|
||||
export function getDependentsSet(map: TaskDependencies, task: Task): Set<Task> {
|
||||
if (!map.has(task)) {
|
||||
map.set(task, new Set());
|
||||
}
|
||||
return map.get(task) !;
|
||||
}
|
||||
|
||||
/**
|
||||
* Invert the given mapping of Task dependencies.
|
||||
*
|
||||
* @param dependencies The mapping of tasks to the tasks that depend upon them.
|
||||
* @returns A mapping of tasks to the tasks that they depend upon.
|
||||
*/
|
||||
export function getBlockedTasks(dependencies: TaskDependencies): Map<Task, Set<Task>> {
|
||||
const blockedTasks = new Map<Task, Set<Task>>();
|
||||
for (const [dependency, dependents] of dependencies) {
|
||||
for (const dependent of dependents) {
|
||||
const dependentSet = getDependentsSet(blockedTasks, dependent);
|
||||
dependentSet.add(dependency);
|
||||
}
|
||||
}
|
||||
return blockedTasks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort a list of tasks by priority.
|
||||
*
|
||||
* Priority is determined by the number of other tasks that a task is (transitively) blocking:
|
||||
* The more tasks a task is blocking the higher its priority is, because processing it will
|
||||
* potentially unblock more tasks.
|
||||
*
|
||||
* To keep the behavior predictable, if two tasks block the same number of other tasks, their
|
||||
* relative order in the original `tasks` lists is preserved.
|
||||
*
|
||||
* @param tasks A (partially ordered) list of tasks.
|
||||
* @param dependencies The mapping of tasks to the tasks that depend upon them.
|
||||
* @return The list of tasks sorted by priority.
|
||||
*/
|
||||
export function sortTasksByPriority(
|
||||
tasks: PartiallyOrderedTasks, dependencies: TaskDependencies): PartiallyOrderedTasks {
|
||||
const priorityPerTask = new Map<Task, [number, number]>();
|
||||
const computePriority = (task: Task, idx: number):
|
||||
[number, number] => [dependencies.has(task) ? dependencies.get(task) !.size : 0, idx];
|
||||
|
||||
tasks.forEach((task, i) => priorityPerTask.set(task, computePriority(task, i)));
|
||||
|
||||
return tasks.slice().sort((task1, task2) => {
|
||||
const [p1, idx1] = priorityPerTask.get(task1) !;
|
||||
const [p2, idx2] = priorityPerTask.get(task2) !;
|
||||
|
||||
return (p2 - p1) || (idx1 - idx2);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import {CreateTaskCompletedCallback, PartiallyOrderedTasks, Task, TaskProcessing
|
|||
import {composeTaskCompletedCallbacks, createMarkAsProcessedHandler, createThrowErrorHandler} from './execution/tasks/completion';
|
||||
import {ParallelTaskQueue} from './execution/tasks/queues/parallel_task_queue';
|
||||
import {SerialTaskQueue} from './execution/tasks/queues/serial_task_queue';
|
||||
import {computeTaskDependencies} from './execution/tasks/utils';
|
||||
import {AsyncLocker} from './locking/async_locker';
|
||||
import {LockFileWithChildProcess} from './locking/lock_file_with_child_process';
|
||||
import {SyncLocker} from './locking/sync_locker';
|
||||
|
|
@ -349,7 +350,9 @@ function getFileWriter(
|
|||
|
||||
function getTaskQueue(
|
||||
inParallel: boolean, tasks: PartiallyOrderedTasks, graph: DepGraph<EntryPoint>): TaskQueue {
|
||||
return inParallel ? new ParallelTaskQueue(tasks, graph) : new SerialTaskQueue(tasks);
|
||||
const dependencies = computeTaskDependencies(tasks, graph);
|
||||
return inParallel ? new ParallelTaskQueue(tasks, dependencies) :
|
||||
new SerialTaskQueue(tasks, dependencies);
|
||||
}
|
||||
|
||||
function getCreateTaskCompletedCallback(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
import {DepGraph} from 'dependency-graph';
|
||||
import {PartiallyOrderedTasks, Task} from '../../src/execution/tasks/api';
|
||||
import {EntryPoint} from '../../src/packages/entry_point';
|
||||
|
||||
/**
|
||||
* Create a set of tasks and a graph of their interdependencies.
|
||||
*
|
||||
* NOTE 1: The first task for each entry-point generates typings (which is similar to what happens
|
||||
* in the actual code).
|
||||
* NOTE 2: The `computeTaskDependencies()` implementation relies on the fact that tasks are sorted in such
|
||||
* a way that a task can only depend upon earlier tasks (i.e. dependencies always come
|
||||
* before dependents in the list of tasks).
|
||||
* To preserve this attribute, you need to ensure that entry-points will only depend on
|
||||
* entry-points with a lower index. Take this into account when defining `entryPointDeps`.
|
||||
* (Failing to do so, will result in an error.)
|
||||
*
|
||||
* @param entryPointCount The number of different entry-points to mock.
|
||||
* @param tasksPerEntryPointCount The number of tasks to generate per entry-point (i.e. simulating
|
||||
* processing multiple format properties).
|
||||
* @param entryPointDeps An object mapping an entry-point to its dependencies. Keys are
|
||||
* entry-point indices and values are arrays of entry-point indices that the
|
||||
* entry-point corresponding to the key depends on.
|
||||
* For example, if entry-point #2 depends on entry-points #0 and #1,
|
||||
* `entryPointDeps` would be `{2: [0, 1]}`.
|
||||
* @return An object with the following properties:
|
||||
* - `tasks`: The (partially ordered) list of generated mock tasks.
|
||||
* - `graph`: The dependency graph for the generated mock entry-point.
|
||||
*/
|
||||
export function createTasksAndGraph(
|
||||
entryPointCount: number, tasksPerEntryPointCount = 1,
|
||||
entryPointDeps: {[entryPointIndex: string]: number[]} = {}):
|
||||
{tasks: PartiallyOrderedTasks, graph: DepGraph<EntryPoint>} {
|
||||
const entryPoints: EntryPoint[] = [];
|
||||
const tasks: PartiallyOrderedTasks = [] as any;
|
||||
const graph = new DepGraph<EntryPoint>();
|
||||
|
||||
// Create the entry-points and the associated tasks.
|
||||
for (let epIdx = 0; epIdx < entryPointCount; epIdx++) {
|
||||
const entryPoint = {
|
||||
name: `entry-point-${epIdx}`,
|
||||
path: `/path/to/entry/point/${epIdx}`,
|
||||
} as EntryPoint;
|
||||
|
||||
entryPoints.push(entryPoint);
|
||||
graph.addNode(entryPoint.path);
|
||||
|
||||
for (let tIdx = 0; tIdx < tasksPerEntryPointCount; tIdx++) {
|
||||
tasks.push({ entryPoint, formatProperty: `prop-${tIdx}`, processDts: tIdx === 0 } as Task);
|
||||
}
|
||||
}
|
||||
|
||||
// Define entry-point interdependencies.
|
||||
for (const epIdx of Object.keys(entryPointDeps).map(strIdx => +strIdx)) {
|
||||
const fromPath = entryPoints[epIdx].path;
|
||||
for (const depIdx of entryPointDeps[epIdx]) {
|
||||
// Ensure that each entry-point only depends on entry-points at a lower index.
|
||||
if (depIdx >= epIdx) {
|
||||
throw Error(
|
||||
'Invalid `entryPointDeps`: Entry-points can only depend on entry-points at a lower ' +
|
||||
`index, but entry-point #${epIdx} depends on #${depIdx} in: ` +
|
||||
JSON.stringify(entryPointDeps, null, 2));
|
||||
}
|
||||
|
||||
const toPath = entryPoints[depIdx].path;
|
||||
graph.addDependency(fromPath, toPath);
|
||||
}
|
||||
}
|
||||
|
||||
return {tasks, graph};
|
||||
}
|
||||
|
|
@ -6,26 +6,16 @@
|
|||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {DepGraph} from 'dependency-graph';
|
||||
|
||||
import {PartiallyOrderedTasks, Task, TaskQueue} from '../../../../src/execution/tasks/api';
|
||||
import {PartiallyOrderedTasks, TaskQueue} from '../../../../src/execution/tasks/api';
|
||||
import {ParallelTaskQueue} from '../../../../src/execution/tasks/queues/parallel_task_queue';
|
||||
import {EntryPoint} from '../../../../src/packages/entry_point';
|
||||
|
||||
import {computeTaskDependencies} from '../../../../src/execution/tasks/utils';
|
||||
import {createTasksAndGraph} from '../../helpers';
|
||||
|
||||
describe('ParallelTaskQueue', () => {
|
||||
// Helpers
|
||||
/**
|
||||
* Create a `TaskQueue` by generating mock tasks (optionally with interdependencies).
|
||||
*
|
||||
* NOTE 1: The first task for each entry-point generates typings (which is similar to what happens
|
||||
* in the actual code).
|
||||
* NOTE 2: The `ParallelTaskQueue` implementation relies on the fact that tasks are sorted in such
|
||||
* a way that a task can only be blocked by earlier tasks (i.e. dependencies always come
|
||||
* before dependants in the list of tasks).
|
||||
* To preserve this attribute, you need to ensure that entry-points will only depend on
|
||||
* entry-points with a lower index. Take this into account when defining `entryPointDeps`.
|
||||
* (Failing to do so, will result in an error.)
|
||||
* See `createTasksAndGraph()` for important usage notes.
|
||||
*
|
||||
* @param entryPointCount The number of different entry-points to mock.
|
||||
* @param tasksPerEntryPointCount The number of tasks to generate per entry-point (i.e. simulating
|
||||
|
|
@ -36,51 +26,18 @@ describe('ParallelTaskQueue', () => {
|
|||
* For example, if entry-point #2 depends on entry-points #0 and #1,
|
||||
* `entryPointDeps` would be `{2: [0, 1]}`.
|
||||
* @return An object with the following properties:
|
||||
* - `graph`: The dependency graph for the generated mock entry-point.
|
||||
* - `tasks`: The (partially ordered) list of generated mock tasks.
|
||||
* - `queue`: The created `TaskQueue`.
|
||||
*/
|
||||
const createQueue = (entryPointCount: number, tasksPerEntryPointCount = 1, entryPointDeps: {
|
||||
[entryPointIndex: string]: number[]
|
||||
} = {}): {tasks: PartiallyOrderedTasks, graph: DepGraph<EntryPoint>, queue: TaskQueue} => {
|
||||
const entryPoints: EntryPoint[] = [];
|
||||
const tasks: PartiallyOrderedTasks = [] as any;
|
||||
const graph = new DepGraph<EntryPoint>();
|
||||
|
||||
// Create the entry-points and the associated tasks.
|
||||
for (let epIdx = 0; epIdx < entryPointCount; epIdx++) {
|
||||
const entryPoint = {
|
||||
name: `entry-point-${epIdx}`,
|
||||
path: `/path/to/entry/point/${epIdx}`,
|
||||
} as EntryPoint;
|
||||
|
||||
entryPoints.push(entryPoint);
|
||||
graph.addNode(entryPoint.path);
|
||||
|
||||
for (let tIdx = 0; tIdx < tasksPerEntryPointCount; tIdx++) {
|
||||
tasks.push({ entryPoint, formatProperty: `prop-${tIdx}`, processDts: tIdx === 0, } as Task);
|
||||
}
|
||||
}
|
||||
|
||||
// Define entry-point interdependencies.
|
||||
for (const epIdx of Object.keys(entryPointDeps).map(strIdx => +strIdx)) {
|
||||
const fromPath = entryPoints[epIdx].path;
|
||||
for (const depIdx of entryPointDeps[epIdx]) {
|
||||
// Ensure that each entry-point only depends on entry-points at a lower index.
|
||||
if (depIdx >= epIdx) {
|
||||
throw Error(
|
||||
'Invalid `entryPointDeps`: Entry-points can only depend on entry-points at a lower ' +
|
||||
`index, but entry-point #${epIdx} depends on #${depIdx} in: ` +
|
||||
JSON.stringify(entryPointDeps, null, 2));
|
||||
}
|
||||
|
||||
const toPath = entryPoints[depIdx].path;
|
||||
graph.addDependency(fromPath, toPath);
|
||||
}
|
||||
}
|
||||
|
||||
return {tasks, graph, queue: new ParallelTaskQueue(tasks.slice(), graph)};
|
||||
};
|
||||
function createQueue(
|
||||
entryPointCount: number, tasksPerEntryPointCount = 1,
|
||||
entryPointDeps: {[entryPointIndex: string]: number[]} = {}):
|
||||
{tasks: PartiallyOrderedTasks, queue: TaskQueue} {
|
||||
const {tasks, graph} =
|
||||
createTasksAndGraph(entryPointCount, tasksPerEntryPointCount, entryPointDeps);
|
||||
const dependencies = computeTaskDependencies(tasks, graph);
|
||||
return {tasks, queue: new ParallelTaskQueue(tasks.slice(), dependencies)};
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulate processing the next task:
|
||||
|
|
@ -149,27 +106,9 @@ describe('ParallelTaskQueue', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('constructor()', () => {
|
||||
it('should throw, if there are multiple tasks that generate typings for a single entry-point',
|
||||
() => {
|
||||
const {tasks, graph} = createQueue(2, 2, {
|
||||
0: [], // Entry-point #0 does not depend on anything.
|
||||
1: [0], // Entry-point #1 depends on #0.
|
||||
});
|
||||
tasks[1].processDts = true; // Tweak task #1 to also generate typings.
|
||||
|
||||
expect(() => new ParallelTaskQueue(tasks, graph))
|
||||
.toThrowError(
|
||||
'Invariant violated: Multiple tasks are assigned generating typings for ' +
|
||||
'\'/path/to/entry/point/0\':\n' +
|
||||
' - {entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-0, formatProperty: prop-1, processDts: true}');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getNextTask()', () => {
|
||||
it('should return the tasks in order (when they are not blocked by other tasks)', () => {
|
||||
const {tasks, queue} = createQueue(3, 2, {}); // 2 tasks per entry-point; no dependencies.
|
||||
const {tasks, queue} = createQueue(6, 1, {}); // 1 task per entry-point; no dependencies.
|
||||
|
||||
expect(queue.getNextTask()).toBe(tasks[0]);
|
||||
expect(queue.getNextTask()).toBe(tasks[1]);
|
||||
|
|
@ -201,26 +140,33 @@ describe('ParallelTaskQueue', () => {
|
|||
|
||||
// Verify that the first two tasks are for the first entry-point.
|
||||
expect(tasks[0].entryPoint.name).toBe('entry-point-0');
|
||||
expect(tasks[0].processDts).toBe(true);
|
||||
expect(tasks[1].entryPoint.name).toBe('entry-point-0');
|
||||
expect(tasks[1].processDts).toBe(false);
|
||||
|
||||
// Verify that the last two tasks are for the second entry-point.
|
||||
expect(tasks[2].entryPoint.name).toBe('entry-point-1');
|
||||
expect(tasks[3].entryPoint.name).toBe('entry-point-1');
|
||||
|
||||
// Return the first two tasks first, since they are not blocked.
|
||||
// Return the first task, since it is not blocked.
|
||||
expect(queue.getNextTask()).toBe(tasks[0]);
|
||||
expect(queue.getNextTask()).toBe(tasks[1]);
|
||||
|
||||
// No task available, until task #0 (which geenrates typings for entry-point #0) is completed.
|
||||
expect(tasks[0].processDts).toBe(true);
|
||||
expect(tasks[1].processDts).toBe(false);
|
||||
|
||||
queue.markTaskCompleted(tasks[1]);
|
||||
// But the rest are blocked on the first task
|
||||
expect(queue.getNextTask()).toBe(null);
|
||||
|
||||
// Finally, unblock tasks for entry-point #1, once task #0 is completed.
|
||||
// Unblock typings task for entry-point #1 and non-typings task for entry-point #0
|
||||
queue.markTaskCompleted(tasks[0]);
|
||||
expect(queue.getNextTask()).toBe(tasks[2]);
|
||||
expect(queue.getNextTask()).toBe(tasks[1]);
|
||||
|
||||
// The non-typings task for entry-point #1 is blocked on the typings task
|
||||
expect(queue.getNextTask()).toBe(null);
|
||||
queue.markTaskCompleted(tasks[1]);
|
||||
// Still blocked because we only completed a non-blocking task
|
||||
expect(queue.getNextTask()).toBe(null);
|
||||
|
||||
// Finally, unblock non-typings task for entry-point #1
|
||||
queue.markTaskCompleted(tasks[2]);
|
||||
expect(queue.getNextTask()).toBe(tasks[3]);
|
||||
});
|
||||
|
||||
|
|
@ -455,34 +401,56 @@ describe('ParallelTaskQueue', () => {
|
|||
|
||||
expect(queue.toString())
|
||||
.toContain(
|
||||
' Blocked tasks (4): \n' +
|
||||
// Tasks #1.0 and #1.1 are blocked by #0.0.
|
||||
' Blocked tasks (6): \n' +
|
||||
// #0.1 blocked by its typings #0.0
|
||||
' - {entryPoint: entry-point-0, formatProperty: prop-1, processDts: false} (1): \n' +
|
||||
' - {entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}\n' +
|
||||
// #1.0 blocked by #0.0
|
||||
' - {entryPoint: entry-point-1, formatProperty: prop-0, processDts: true} (1): \n' +
|
||||
' - {entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-1, formatProperty: prop-1, processDts: false} (1): \n' +
|
||||
// #1.1 blocked by #0.0 and its typings #1.0
|
||||
' - {entryPoint: entry-point-1, formatProperty: prop-1, processDts: false} (2): \n' +
|
||||
' - {entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}\n' +
|
||||
// Tasks #3.0 and #3.1 are blocked by #0.0 (transitively), #1.0 and #2.0.
|
||||
' - {entryPoint: entry-point-1, formatProperty: prop-0, processDts: true}\n' +
|
||||
// #3.0 blocked by #0.0 (transitively), #1.0 and #2.0.
|
||||
' - {entryPoint: entry-point-3, formatProperty: prop-0, processDts: true} (3): \n' +
|
||||
' - {entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-1, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-2, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-3, formatProperty: prop-1, processDts: false} (3): \n' +
|
||||
// #3.1 blocked by #0.0 (transitively), #1.0 and #2.0, and its typings #3.0
|
||||
' - {entryPoint: entry-point-3, formatProperty: prop-1, processDts: false} (4): \n' +
|
||||
' - {entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-1, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-2, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-3, formatProperty: prop-0, processDts: true}\n' +
|
||||
// #2.1 blocked by its typings #2.0
|
||||
' - {entryPoint: entry-point-2, formatProperty: prop-1, processDts: false} (1): \n' +
|
||||
' - {entryPoint: entry-point-2, formatProperty: prop-0, processDts: true}');
|
||||
|
||||
expect(processNextTask(queue)).toBe(tasks[0]); // Process #0.0.
|
||||
expect(processNextTask(queue)).toBe(tasks[2]); // Process #1.0.
|
||||
expect(queue.toString())
|
||||
.toContain(
|
||||
' Blocked tasks (2): \n' +
|
||||
// Tasks #3.0 and #3.1 are blocked by #2.0.
|
||||
' Blocked tasks (3): \n' +
|
||||
// #3.0 blocked by #2.0.
|
||||
' - {entryPoint: entry-point-3, formatProperty: prop-0, processDts: true} (1): \n' +
|
||||
' - {entryPoint: entry-point-2, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-3, formatProperty: prop-1, processDts: false} (1): \n' +
|
||||
// #3.1 blocked by #2.0 and its typings #3.0
|
||||
' - {entryPoint: entry-point-3, formatProperty: prop-1, processDts: false} (2): \n' +
|
||||
' - {entryPoint: entry-point-2, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-3, formatProperty: prop-0, processDts: true}\n' +
|
||||
// #2.1 blocked by its typings #2.0
|
||||
' - {entryPoint: entry-point-2, formatProperty: prop-1, processDts: false} (1): \n' +
|
||||
' - {entryPoint: entry-point-2, formatProperty: prop-0, processDts: true}');
|
||||
|
||||
expect(processNextTask(queue)).toBe(tasks[4]); // Process #2.0.
|
||||
expect(queue.toString())
|
||||
.toContain(
|
||||
' Blocked tasks (1): \n' +
|
||||
// #3.1 blocked by its typings #3.0
|
||||
' - {entryPoint: entry-point-3, formatProperty: prop-1, processDts: false} (1): \n' +
|
||||
' - {entryPoint: entry-point-3, formatProperty: prop-0, processDts: true}');
|
||||
expect(processNextTask(queue)).toBe(tasks[6]); // Process #3.0.
|
||||
expect(queue.toString()).toContain(' Blocked tasks (0): ');
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -5,9 +5,12 @@
|
|||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
import {DepGraph} from 'dependency-graph';
|
||||
|
||||
import {PartiallyOrderedTasks, Task, TaskQueue} from '../../../../src/execution/tasks/api';
|
||||
import {SerialTaskQueue} from '../../../../src/execution/tasks/queues/serial_task_queue';
|
||||
import {computeTaskDependencies} from '../../../../src/execution/tasks/utils';
|
||||
import {EntryPoint} from '../../../../src/packages/entry_point';
|
||||
|
||||
|
||||
describe('SerialTaskQueue', () => {
|
||||
|
|
@ -24,13 +27,18 @@ describe('SerialTaskQueue', () => {
|
|||
*/
|
||||
const createQueue = (taskCount: number): {tasks: PartiallyOrderedTasks, queue: TaskQueue} => {
|
||||
const tasks: PartiallyOrderedTasks = [] as any;
|
||||
const graph = new DepGraph<EntryPoint>();
|
||||
for (let i = 0; i < taskCount; i++) {
|
||||
tasks.push({
|
||||
entryPoint: {name: `entry-point-${i}`}, formatProperty: `prop-${i}`,
|
||||
processDts: i % 2 === 0,
|
||||
} as Task);
|
||||
const entryPoint = {
|
||||
name: `entry-point-${i}`,
|
||||
path: `/path/to/entry/point/${i}`
|
||||
} as EntryPoint;
|
||||
tasks.push(
|
||||
{ entryPoint: entryPoint, formatProperty: `prop-${i}`, processDts: i % 2 === 0 } as Task);
|
||||
graph.addNode(entryPoint.path);
|
||||
}
|
||||
return {tasks, queue: new SerialTaskQueue(tasks.slice())};
|
||||
const dependencies = computeTaskDependencies(tasks, graph);
|
||||
return {tasks, queue: new SerialTaskQueue(tasks.slice(), dependencies)};
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
import {computeTaskDependencies, sortTasksByPriority} from '../../src/execution/tasks/utils';
|
||||
import {createTasksAndGraph} from './helpers';
|
||||
|
||||
describe('execution utils', () => {
|
||||
describe('computeTaskDependencies()', () => {
|
||||
it('should throw, if there are multiple tasks that generate typings for a single entry-point',
|
||||
() => {
|
||||
const {tasks, graph} = createTasksAndGraph(2, 2, {
|
||||
0: [], // Entry-point #0 does not depend on anything.
|
||||
1: [0], // Entry-point #1 depends on #0.
|
||||
});
|
||||
tasks[1].processDts = true; // Tweak task #1 to also generate typings.
|
||||
|
||||
expect(() => computeTaskDependencies(tasks, graph))
|
||||
.toThrowError(
|
||||
'Invariant violated: Multiple tasks are assigned generating typings for ' +
|
||||
'\'/path/to/entry/point/0\':\n' +
|
||||
' - {entryPoint: entry-point-0, formatProperty: prop-0, processDts: true}\n' +
|
||||
' - {entryPoint: entry-point-0, formatProperty: prop-1, processDts: true}');
|
||||
});
|
||||
|
||||
it('should add non-typings tasks to the dependents of typings tasks', () => {
|
||||
const {tasks, graph} = createTasksAndGraph(2, 2, {
|
||||
0: [], // entry-point-0 does not depend on anything.
|
||||
1: [0], // entry-point-1 depends on entry-point-0.
|
||||
});
|
||||
|
||||
const dependents = computeTaskDependencies(tasks, graph);
|
||||
// entry-point-0
|
||||
expect(dependents.get(tasks[0])).toEqual(new Set([
|
||||
tasks[1], // non-typings task for entry-point-0
|
||||
tasks[2], // typings task for entry-point-1, which depends upon entry-point-0
|
||||
tasks[3], // non-typings task for entry-point-1, which depends upon entry-point-0
|
||||
]));
|
||||
expect(dependents.get(tasks[1])).toBeUndefined();
|
||||
|
||||
// entry-point-1
|
||||
expect(dependents.get(tasks[2])).toEqual(new Set([
|
||||
tasks[3], // non-typings task for entry-point-1
|
||||
]));
|
||||
expect(dependents.get(tasks[3])).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('sortTasksByPriority', () => {
|
||||
it('should return the tasks in their original order if there are no dependencies', () => {
|
||||
const {tasks} = createTasksAndGraph(3, 1);
|
||||
const sortedTasks = sortTasksByPriority(tasks, new Map());
|
||||
expect(sortedTasks).toEqual(tasks);
|
||||
});
|
||||
|
||||
it('should return the tasks ordered by how many tasks depend upon them', () => {
|
||||
// Before sort:
|
||||
// 0 blocks [3]
|
||||
// 1 blocks [2, 3]
|
||||
// 2 blocks []
|
||||
// 3 blocks [4]
|
||||
// 4 blocks []
|
||||
const {tasks, graph} = createTasksAndGraph(5, 1, {0: [], 1: [], 2: [1], 3: [0, 1], 4: [3]});
|
||||
const sortedTasks = sortTasksByPriority(tasks, computeTaskDependencies(tasks, graph));
|
||||
// After sort:
|
||||
// 1 blocks [2, 3]
|
||||
// 0 blocks [3]
|
||||
// 3 blocks [4]
|
||||
// 2 blocks []
|
||||
// 4 blocks []
|
||||
expect(sortedTasks).toEqual([
|
||||
tasks[1],
|
||||
tasks[0],
|
||||
tasks[3],
|
||||
tasks[2],
|
||||
tasks[4],
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue