perf(ngcc): process tasks in parallel in async mode (#32427)
`ngcc` supports both synchronous and asynchronous execution. The default mode when using `ngcc` programmatically (which is how `@angular/cli` is using it) is synchronous. When running `ngcc` from the command line (i.e. via the `ivy-ngcc` script), it runs in async mode. Previously, the work would be executed in the same way in both modes. This commit improves the performance of `ngcc` in async mode by processing tasks in parallel on multiple processes. It uses the Node.js built-in [`cluster` module](https://nodejs.org/api/cluster.html) to launch a cluster of Node.js processes and take advantage of multi-core systems. Preliminary comparisons indicate a 1.8x to 2.6x speed improvement when processing the angular.io app (apparently depending on the OS, number of available cores, system load, etc.). Further investigation is needed to better understand these numbers and identify potential areas of improvement. Inspired by/Based on @alxhub's prototype: alxhub/angular@cb631bdb1 Original design doc: https://hackmd.io/uYG9CJrFQZ-6FtKqpnYJAA?view Jira issue: [FW-1460](https://angular-team.atlassian.net/browse/FW-1460) PR Close #32427
This commit is contained in:
George Kalpakas
Matias Niemelä
parent
f4e4bb2085
commit
e36e6c85ef
|
|
@ -82,6 +82,10 @@ grep "FocusMonitor.decorators =" node_modules/@angular/cdk/bundles/cdk-a11y.umd.
|
|||
ivy-ngcc -l debug | grep 'Skipping'
|
||||
if [[ $? != 0 ]]; then exit 1; fi
|
||||
|
||||
# Does it process the tasks in parallel?
|
||||
ivy-ngcc -l debug | grep 'Running ngcc on ClusterExecutor'
|
||||
if [[ $? != 0 ]]; then exit 1; fi
|
||||
|
||||
# Check that running it with logging level error outputs nothing
|
||||
ivy-ngcc -l error | grep '.' && exit 1
|
||||
|
||||
|
|
|
|||
|
|
@ -63,16 +63,17 @@ export interface DependencyDiagnostics {
|
|||
export type PartiallyOrderedEntryPoints = PartiallyOrderedList<EntryPoint>;
|
||||
|
||||
/**
|
||||
* A list of entry-points, sorted by their dependencies.
|
||||
* A list of entry-points, sorted by their dependencies, and the dependency graph.
|
||||
*
|
||||
* The `entryPoints` array will be ordered so that no entry point depends upon an entry point that
|
||||
* appears later in the array.
|
||||
*
|
||||
* Some entry points or their dependencies may be have been ignored. These are captured for
|
||||
* Some entry points or their dependencies may have been ignored. These are captured for
|
||||
* diagnostic purposes in `invalidEntryPoints` and `ignoredDependencies` respectively.
|
||||
*/
|
||||
export interface SortedEntryPointsInfo extends DependencyDiagnostics {
|
||||
entryPoints: PartiallyOrderedEntryPoints;
|
||||
graph: DepGraph<EntryPoint>;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -109,6 +110,7 @@ export class DependencyResolver {
|
|||
return {
|
||||
entryPoints: (sortedEntryPointNodes as PartiallyOrderedList<string>)
|
||||
.map(path => graph.getNodeData(path)),
|
||||
graph,
|
||||
invalidEntryPoints,
|
||||
ignoredDependencies,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {EntryPoint, EntryPointJsonProperty} from '../packages/entry_point';
|
||||
import {EntryPoint, EntryPointJsonProperty, JsonObject} from '../packages/entry_point';
|
||||
import {PartiallyOrderedList} from '../utils';
|
||||
|
||||
|
||||
|
|
@ -49,7 +49,7 @@ export interface Executor {
|
|||
export type PartiallyOrderedTasks = PartiallyOrderedList<Task>;
|
||||
|
||||
/** Represents a unit of work: processing a specific format property of an entry-point. */
|
||||
export interface Task {
|
||||
export interface Task extends JsonObject {
|
||||
/** The `EntryPoint` which needs to be processed as part of the task. */
|
||||
entryPoint: EntryPoint;
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {AbsoluteFsPath} from '../../../../src/ngtsc/file_system';
|
||||
import {JsonObject} from '../../packages/entry_point';
|
||||
import {PackageJsonChange} from '../../writing/package_json_updater';
|
||||
import {Task, TaskProcessingOutcome} from '../api';
|
||||
|
||||
|
||||
/** A message reporting that an unrecoverable error occurred. */
|
||||
export interface ErrorMessage extends JsonObject {
|
||||
type: 'error';
|
||||
error: string;
|
||||
}
|
||||
|
||||
/** A message requesting the processing of a task. */
|
||||
export interface ProcessTaskMessage extends JsonObject {
|
||||
type: 'process-task';
|
||||
task: Task;
|
||||
}
|
||||
|
||||
/**
|
||||
* A message reporting the result of processing the currently assigned task.
|
||||
*
|
||||
* NOTE: To avoid the communication overhead, the task is not included in the message. Instead, the
|
||||
* master is responsible for keeping a mapping of workers to their currently assigned tasks.
|
||||
*/
|
||||
export interface TaskCompletedMessage extends JsonObject {
|
||||
type: 'task-completed';
|
||||
outcome: TaskProcessingOutcome;
|
||||
}
|
||||
|
||||
/** A message requesting the update of a `package.json` file. */
|
||||
export interface UpdatePackageJsonMessage extends JsonObject {
|
||||
type: 'update-package-json';
|
||||
packageJsonPath: AbsoluteFsPath;
|
||||
changes: PackageJsonChange[];
|
||||
}
|
||||
|
||||
/** The type of messages sent from cluster workers to the cluster master. */
|
||||
export type MessageFromWorker = ErrorMessage | TaskCompletedMessage | UpdatePackageJsonMessage;
|
||||
|
||||
/** The type of messages sent from the cluster master to cluster workers. */
|
||||
export type MessageToWorker = ProcessTaskMessage;
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/// <reference types="node" />
|
||||
|
||||
import * as cluster from 'cluster';
|
||||
|
||||
import {Logger} from '../../logging/logger';
|
||||
import {PackageJsonUpdater} from '../../writing/package_json_updater';
|
||||
import {AnalyzeEntryPointsFn, CreateCompileFn, Executor} from '../api';
|
||||
|
||||
import {ClusterMaster} from './master';
|
||||
import {ClusterWorker} from './worker';
|
||||
|
||||
|
||||
/**
|
||||
* An `Executor` that processes tasks in parallel (on multiple processes) and completes
|
||||
* asynchronously.
|
||||
*/
|
||||
export class ClusterExecutor implements Executor {
|
||||
constructor(
|
||||
private workerCount: number, private logger: Logger,
|
||||
private pkgJsonUpdater: PackageJsonUpdater) {}
|
||||
|
||||
async execute(analyzeEntryPoints: AnalyzeEntryPointsFn, createCompileFn: CreateCompileFn):
|
||||
Promise<void> {
|
||||
if (cluster.isMaster) {
|
||||
this.logger.debug(
|
||||
`Running ngcc on ${this.constructor.name} (using ${this.workerCount} worker processes).`);
|
||||
|
||||
// This process is the cluster master.
|
||||
const master =
|
||||
new ClusterMaster(this.workerCount, this.logger, this.pkgJsonUpdater, analyzeEntryPoints);
|
||||
return master.run();
|
||||
} else {
|
||||
// This process is a cluster worker.
|
||||
const worker = new ClusterWorker(createCompileFn);
|
||||
return worker.run();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,253 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/// <reference types="node" />
|
||||
|
||||
import * as cluster from 'cluster';
|
||||
|
||||
import {resolve} from '../../../../src/ngtsc/file_system';
|
||||
import {Logger} from '../../logging/logger';
|
||||
import {PackageJsonUpdater} from '../../writing/package_json_updater';
|
||||
import {AnalyzeEntryPointsFn, Task, TaskQueue} from '../api';
|
||||
import {onTaskCompleted, stringifyTask} from '../utils';
|
||||
|
||||
import {MessageFromWorker, TaskCompletedMessage, UpdatePackageJsonMessage} from './api';
|
||||
import {Deferred, sendMessageToWorker} from './utils';
|
||||
|
||||
|
||||
/**
|
||||
* The cluster master is responsible for analyzing all entry-points, planning the work that needs to
|
||||
* be done, distributing it to worker-processes and collecting/post-processing the results.
|
||||
*/
|
||||
export class ClusterMaster {
|
||||
private finishedDeferred = new Deferred<void>();
|
||||
private taskAssignments = new Map<number, Task|null>();
|
||||
private taskQueue: TaskQueue;
|
||||
|
||||
constructor(
|
||||
private workerCount: number, private logger: Logger,
|
||||
private pkgJsonUpdater: PackageJsonUpdater, analyzeEntryPoints: AnalyzeEntryPointsFn) {
|
||||
if (!cluster.isMaster) {
|
||||
throw new Error('Tried to instantiate `ClusterMaster` on a worker process.');
|
||||
}
|
||||
|
||||
this.taskQueue = analyzeEntryPoints();
|
||||
}
|
||||
|
||||
run(): Promise<void> {
|
||||
// Set up listeners for worker events (emitted on `cluster`).
|
||||
cluster.on('online', this.wrapEventHandler(worker => this.onWorkerOnline(worker.id)));
|
||||
|
||||
cluster.on(
|
||||
'message', this.wrapEventHandler((worker, msg) => this.onWorkerMessage(worker.id, msg)));
|
||||
|
||||
cluster.on(
|
||||
'exit',
|
||||
this.wrapEventHandler((worker, code, signal) => this.onWorkerExit(worker, code, signal)));
|
||||
|
||||
// Start the workers.
|
||||
for (let i = 0; i < this.workerCount; i++) {
|
||||
cluster.fork();
|
||||
}
|
||||
|
||||
return this.finishedDeferred.promise.then(() => this.stopWorkers(), err => {
|
||||
this.stopWorkers();
|
||||
return Promise.reject(err);
|
||||
});
|
||||
}
|
||||
|
||||
/** Try to find available (idle) workers and assign them available (non-blocked) tasks. */
|
||||
private maybeDistributeWork(): void {
|
||||
let isWorkerAvailable = false;
|
||||
|
||||
// First, check whether all tasks have been completed.
|
||||
if (this.taskQueue.allTasksCompleted) {
|
||||
return this.finishedDeferred.resolve();
|
||||
}
|
||||
|
||||
// Look for available workers and available tasks to assign to them.
|
||||
for (const [workerId, assignedTask] of Array.from(this.taskAssignments)) {
|
||||
if (assignedTask !== null) {
|
||||
// This worker already has a job; check other workers.
|
||||
continue;
|
||||
} else {
|
||||
// This worker is available.
|
||||
isWorkerAvailable = true;
|
||||
}
|
||||
|
||||
// This worker needs a job. See if any are available.
|
||||
const task = this.taskQueue.getNextTask();
|
||||
if (task === null) {
|
||||
// No suitable work available right now.
|
||||
break;
|
||||
}
|
||||
|
||||
// Process the next task on the worker.
|
||||
this.taskAssignments.set(workerId, task);
|
||||
sendMessageToWorker(workerId, {type: 'process-task', task});
|
||||
|
||||
isWorkerAvailable = false;
|
||||
}
|
||||
|
||||
// If there are no available workers or no available tasks, log (for debugging purposes).
|
||||
if (!isWorkerAvailable) {
|
||||
this.logger.debug(
|
||||
`All ${this.taskAssignments.size} workers are currently busy and cannot take on more ` +
|
||||
'work.');
|
||||
} else {
|
||||
const busyWorkers = Array.from(this.taskAssignments)
|
||||
.filter(([_workerId, task]) => task !== null)
|
||||
.map(([workerId]) => workerId);
|
||||
const totalWorkerCount = this.taskAssignments.size;
|
||||
const idleWorkerCount = totalWorkerCount - busyWorkers.length;
|
||||
|
||||
this.logger.debug(
|
||||
`No assignments for ${idleWorkerCount} idle (out of ${totalWorkerCount} total) ` +
|
||||
`workers. Busy workers: ${busyWorkers.join(', ')}`);
|
||||
|
||||
if (busyWorkers.length === 0) {
|
||||
// This is a bug:
|
||||
// All workers are idle (meaning no tasks are in progress) and `taskQueue.allTasksCompleted`
|
||||
// is `false`, but there is still no assignable work.
|
||||
throw new Error(
|
||||
'There are still unprocessed tasks in the queue and no tasks are currently in ' +
|
||||
`progress, yet the queue did not return any available tasks: ${this.taskQueue}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Handle a worker's exiting. (Might be intentional or not.) */
|
||||
private onWorkerExit(worker: cluster.Worker, code: number|null, signal: string|null): void {
|
||||
// If the worker's exiting was intentional, nothing to do.
|
||||
if (worker.exitedAfterDisconnect) return;
|
||||
|
||||
// The worker exited unexpectedly: Determine it's status and take an appropriate action.
|
||||
const currentTask = this.taskAssignments.get(worker.id);
|
||||
|
||||
this.logger.warn(
|
||||
`Worker #${worker.id} exited unexpectedly (code: ${code} | signal: ${signal}).\n` +
|
||||
` Current assignment: ${(currentTask == null) ? '-' : stringifyTask(currentTask)}`);
|
||||
|
||||
if (currentTask == null) {
|
||||
// The crashed worker process was not in the middle of a task:
|
||||
// Just spawn another process.
|
||||
this.logger.debug(`Spawning another worker process to replace #${worker.id}...`);
|
||||
this.taskAssignments.delete(worker.id);
|
||||
cluster.fork();
|
||||
} else {
|
||||
// The crashed worker process was in the middle of a task:
|
||||
// Impossible to know whether we can recover (without ending up with a corrupted entry-point).
|
||||
throw new Error(
|
||||
'Process unexpectedly crashed, while processing format property ' +
|
||||
`${currentTask.formatProperty} for entry-point '${currentTask.entryPoint.path}'.`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Handle a message from a worker. */
|
||||
private onWorkerMessage(workerId: number, msg: MessageFromWorker): void {
|
||||
if (!this.taskAssignments.has(workerId)) {
|
||||
const knownWorkers = Array.from(this.taskAssignments.keys());
|
||||
throw new Error(
|
||||
`Received message from unknown worker #${workerId} (known workers: ` +
|
||||
`${knownWorkers.join(', ')}): ${JSON.stringify(msg)}`);
|
||||
}
|
||||
|
||||
switch (msg.type) {
|
||||
case 'error':
|
||||
throw new Error(`Error on worker #${workerId}: ${msg.error}`);
|
||||
case 'task-completed':
|
||||
return this.onWorkerTaskCompleted(workerId, msg);
|
||||
case 'update-package-json':
|
||||
return this.onWorkerUpdatePackageJson(workerId, msg);
|
||||
default:
|
||||
throw new Error(
|
||||
`Invalid message received from worker #${workerId}: ${JSON.stringify(msg)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Handle a worker's coming online. */
|
||||
private onWorkerOnline(workerId: number): void {
|
||||
if (this.taskAssignments.has(workerId)) {
|
||||
throw new Error(`Invariant violated: Worker #${workerId} came online more than once.`);
|
||||
}
|
||||
|
||||
this.taskAssignments.set(workerId, null);
|
||||
this.maybeDistributeWork();
|
||||
}
|
||||
|
||||
/** Handle a worker's having completed their assigned task. */
|
||||
private onWorkerTaskCompleted(workerId: number, msg: TaskCompletedMessage): void {
|
||||
const task = this.taskAssignments.get(workerId) || null;
|
||||
|
||||
if (task === null) {
|
||||
throw new Error(
|
||||
`Expected worker #${workerId} to have a task assigned, while handling message: ` +
|
||||
JSON.stringify(msg));
|
||||
}
|
||||
|
||||
onTaskCompleted(this.pkgJsonUpdater, task, msg.outcome);
|
||||
|
||||
this.taskQueue.markTaskCompleted(task);
|
||||
this.taskAssignments.set(workerId, null);
|
||||
this.maybeDistributeWork();
|
||||
}
|
||||
|
||||
/** Handle a worker's request to update a `package.json` file. */
|
||||
private onWorkerUpdatePackageJson(workerId: number, msg: UpdatePackageJsonMessage): void {
|
||||
const task = this.taskAssignments.get(workerId) || null;
|
||||
|
||||
if (task === null) {
|
||||
throw new Error(
|
||||
`Expected worker #${workerId} to have a task assigned, while handling message: ` +
|
||||
JSON.stringify(msg));
|
||||
}
|
||||
|
||||
const expectedPackageJsonPath = resolve(task.entryPoint.path, 'package.json');
|
||||
const parsedPackageJson = task.entryPoint.packageJson;
|
||||
|
||||
if (expectedPackageJsonPath !== msg.packageJsonPath) {
|
||||
throw new Error(
|
||||
`Received '${msg.type}' message from worker #${workerId} for '${msg.packageJsonPath}', ` +
|
||||
`but was expecting '${expectedPackageJsonPath}' (based on task assignment).`);
|
||||
}
|
||||
|
||||
// NOTE: Although the change in the parsed `package.json` will be reflected in tasks objects
|
||||
// locally and thus also in future `process-task` messages sent to worker processes, any
|
||||
// processes already running and processing a task for the same entry-point will not get
|
||||
// the change.
|
||||
// Do not rely on having an up-to-date `package.json` representation in worker processes.
|
||||
// In other words, task processing should only rely on the info that was there when the
|
||||
// file was initially parsed (during entry-point analysis) and not on the info that might
|
||||
// be added later (during task processing).
|
||||
this.pkgJsonUpdater.writeChanges(msg.changes, msg.packageJsonPath, parsedPackageJson);
|
||||
}
|
||||
|
||||
/** Stop all workers and stop listening on cluster events. */
|
||||
private stopWorkers(): void {
|
||||
const workers = Object.values(cluster.workers) as cluster.Worker[];
|
||||
this.logger.debug(`Stopping ${workers.length} workers...`);
|
||||
|
||||
cluster.removeAllListeners();
|
||||
workers.forEach(worker => worker.kill());
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an event handler to ensure that `finishedDeferred` will be rejected on error (regardless
|
||||
* if the handler completes synchronously or asynchronously).
|
||||
*/
|
||||
private wrapEventHandler<Args extends unknown[]>(fn: (...args: Args) => void|Promise<void>):
|
||||
(...args: Args) => Promise<void> {
|
||||
return async(...args: Args) => {
|
||||
try {
|
||||
await fn(...args);
|
||||
} catch (err) {
|
||||
this.finishedDeferred.reject(err);
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/// <reference types="node" />
|
||||
|
||||
import * as cluster from 'cluster';
|
||||
|
||||
import {AbsoluteFsPath} from '../../../../src/ngtsc/file_system';
|
||||
import {JsonObject} from '../../packages/entry_point';
|
||||
import {PackageJsonChange, PackageJsonUpdate, PackageJsonUpdater, applyChange} from '../../writing/package_json_updater';
|
||||
|
||||
import {sendMessageToMaster} from './utils';
|
||||
|
||||
|
||||
/**
|
||||
* A `PackageJsonUpdater` that can safely handle update operations on multiple processes.
|
||||
*/
|
||||
export class ClusterPackageJsonUpdater implements PackageJsonUpdater {
|
||||
constructor(private delegate: PackageJsonUpdater) {}
|
||||
|
||||
createUpdate(): PackageJsonUpdate {
|
||||
return new PackageJsonUpdate((...args) => this.writeChanges(...args));
|
||||
}
|
||||
|
||||
writeChanges(
|
||||
changes: PackageJsonChange[], packageJsonPath: AbsoluteFsPath,
|
||||
preExistingParsedJson?: JsonObject): void {
|
||||
if (cluster.isMaster) {
|
||||
// This is the master process:
|
||||
// Actually apply the changes to the file on disk.
|
||||
return this.delegate.writeChanges(changes, packageJsonPath, preExistingParsedJson);
|
||||
}
|
||||
|
||||
// This is a worker process:
|
||||
// Apply the changes in-memory (if necessary) and send a message to the master process.
|
||||
if (preExistingParsedJson) {
|
||||
for (const [propPath, value] of changes) {
|
||||
if (propPath.length === 0) {
|
||||
throw new Error(`Missing property path for writing value to '${packageJsonPath}'.`);
|
||||
}
|
||||
|
||||
applyChange(preExistingParsedJson, propPath, value);
|
||||
}
|
||||
}
|
||||
|
||||
sendMessageToMaster({
|
||||
type: 'update-package-json',
|
||||
packageJsonPath,
|
||||
changes,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/// <reference types="node" />
|
||||
|
||||
import * as cluster from 'cluster';
|
||||
|
||||
import {MessageFromWorker, MessageToWorker} from './api';
|
||||
|
||||
|
||||
|
||||
/** Expose a `Promise` instance as well as APIs for resolving/rejecting it. */
|
||||
export class Deferred<T> {
|
||||
/**
|
||||
* Resolve the associated promise with the specified value.
|
||||
* If the value is a rejection (constructed with `Promise.reject()`), the promise will be rejected
|
||||
* instead.
|
||||
*
|
||||
* @param value The value to resolve the promise with.
|
||||
*/
|
||||
resolve !: (value: T) => void;
|
||||
|
||||
/**
|
||||
* Rejects the associated promise with the specified reason.
|
||||
*
|
||||
* @param reason The rejection reason.
|
||||
*/
|
||||
reject !: (reason: any) => void;
|
||||
|
||||
/** The `Promise` instance associated with this deferred. */
|
||||
promise = new Promise<T>((resolve, reject) => {
|
||||
this.resolve = resolve;
|
||||
this.reject = reject;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a message to the cluster master.
|
||||
* (This function should be invoked from cluster workers only.)
|
||||
*
|
||||
* @param msg The message to send to the cluster master.
|
||||
*/
|
||||
export const sendMessageToMaster = (msg: MessageFromWorker): void => {
|
||||
if (cluster.isMaster) {
|
||||
throw new Error('Unable to send message to the master process: Already on the master process.');
|
||||
}
|
||||
|
||||
if (process.send === undefined) {
|
||||
// Theoretically, this should never happen on a worker process.
|
||||
throw new Error('Unable to send message to the master process: Missing `process.send()`.');
|
||||
}
|
||||
|
||||
process.send(msg);
|
||||
};
|
||||
|
||||
/**
|
||||
* Send a message to a cluster worker.
|
||||
* (This function should be invoked from the cluster master only.)
|
||||
*
|
||||
* @param workerId The ID of the recipient worker.
|
||||
* @param msg The message to send to the worker.
|
||||
*/
|
||||
export const sendMessageToWorker = (workerId: number, msg: MessageToWorker): void => {
|
||||
if (!cluster.isMaster) {
|
||||
throw new Error('Unable to send message to worker process: Sender is not the master process.');
|
||||
}
|
||||
|
||||
const worker = cluster.workers[workerId];
|
||||
|
||||
if ((worker === undefined) || worker.isDead() || !worker.isConnected()) {
|
||||
throw new Error(
|
||||
'Unable to send message to worker process: Recipient does not exist or has disconnected.');
|
||||
}
|
||||
|
||||
worker.send(msg);
|
||||
};
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/// <reference types="node" />
|
||||
|
||||
import * as cluster from 'cluster';
|
||||
|
||||
import {CompileFn, CreateCompileFn} from '../api';
|
||||
|
||||
import {MessageToWorker} from './api';
|
||||
import {sendMessageToMaster} from './utils';
|
||||
|
||||
|
||||
/**
|
||||
* A cluster worker is responsible for processing one task (i.e. one format property for a specific
|
||||
* entry-point) at a time and reporting results back to the cluster master.
|
||||
*/
|
||||
export class ClusterWorker {
|
||||
private compile: CompileFn;
|
||||
|
||||
constructor(createCompileFn: CreateCompileFn) {
|
||||
if (cluster.isMaster) {
|
||||
throw new Error('Tried to instantiate `ClusterWorker` on the master process.');
|
||||
}
|
||||
|
||||
this.compile =
|
||||
createCompileFn((_task, outcome) => sendMessageToMaster({type: 'task-completed', outcome}));
|
||||
}
|
||||
|
||||
run(): Promise<void> {
|
||||
// Listen for `ProcessTaskMessage`s and process tasks.
|
||||
cluster.worker.on('message', (msg: MessageToWorker) => {
|
||||
try {
|
||||
switch (msg.type) {
|
||||
case 'process-task':
|
||||
return this.compile(msg.task);
|
||||
default:
|
||||
throw new Error(
|
||||
`Invalid message received on worker #${cluster.worker.id}: ${JSON.stringify(msg)}`);
|
||||
}
|
||||
} catch (err) {
|
||||
sendMessageToMaster({
|
||||
type: 'error',
|
||||
error: (err instanceof Error) ? (err.stack || err.message) : err,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Return a promise that is never resolved.
|
||||
return new Promise(() => undefined);
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,11 @@
|
|||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/// <reference types="node" />
|
||||
|
||||
import {DepGraph} from 'dependency-graph';
|
||||
import * as os from 'os';
|
||||
import * as ts from 'typescript';
|
||||
|
||||
import {AbsoluteFsPath, FileSystem, absoluteFrom, dirname, getFileSystem, resolve} from '../../src/ngtsc/file_system';
|
||||
|
|
@ -16,14 +21,17 @@ import {ModuleResolver} from './dependencies/module_resolver';
|
|||
import {UmdDependencyHost} from './dependencies/umd_dependency_host';
|
||||
import {DirectoryWalkerEntryPointFinder} from './entry_point_finder/directory_walker_entry_point_finder';
|
||||
import {TargetedEntryPointFinder} from './entry_point_finder/targeted_entry_point_finder';
|
||||
import {AnalyzeEntryPointsFn, CreateCompileFn, Executor, PartiallyOrderedTasks, Task, TaskProcessingOutcome} from './execution/api';
|
||||
import {AnalyzeEntryPointsFn, CreateCompileFn, Executor, PartiallyOrderedTasks, Task, TaskProcessingOutcome, TaskQueue} from './execution/api';
|
||||
import {ClusterExecutor} from './execution/cluster/executor';
|
||||
import {ClusterPackageJsonUpdater} from './execution/cluster/package_json_updater';
|
||||
import {AsyncSingleProcessExecutor, SingleProcessExecutor} from './execution/single_process_executor';
|
||||
import {ParallelTaskQueue} from './execution/task_selection/parallel_task_queue';
|
||||
import {SerialTaskQueue} from './execution/task_selection/serial_task_queue';
|
||||
import {ConsoleLogger, LogLevel} from './logging/console_logger';
|
||||
import {Logger} from './logging/logger';
|
||||
import {hasBeenProcessed, markAsProcessed} from './packages/build_marker';
|
||||
import {NgccConfiguration} from './packages/configuration';
|
||||
import {EntryPointJsonProperty, EntryPointPackageJson, SUPPORTED_FORMAT_PROPERTIES, getEntryPointFormat} from './packages/entry_point';
|
||||
import {EntryPoint, EntryPointJsonProperty, EntryPointPackageJson, SUPPORTED_FORMAT_PROPERTIES, getEntryPointFormat} from './packages/entry_point';
|
||||
import {makeEntryPointBundle} from './packages/entry_point_bundle';
|
||||
import {Transformer} from './packages/transformer';
|
||||
import {PathMappings} from './utils';
|
||||
|
|
@ -32,6 +40,7 @@ import {InPlaceFileWriter} from './writing/in_place_file_writer';
|
|||
import {NewEntryPointFileWriter} from './writing/new_entry_point_file_writer';
|
||||
import {DirectPackageJsonUpdater, PackageJsonUpdater} from './writing/package_json_updater';
|
||||
|
||||
|
||||
/**
|
||||
* The options to configure the ngcc compiler for synchronous execution.
|
||||
*/
|
||||
|
|
@ -100,6 +109,8 @@ export type AsyncNgccOptions = Omit<SyncNgccOptions, 'async'>& {async: true};
|
|||
*/
|
||||
export type NgccOptions = AsyncNgccOptions | SyncNgccOptions;
|
||||
|
||||
const EMPTY_GRAPH = new DepGraph<EntryPoint>();
|
||||
|
||||
/**
|
||||
* This is the main entry-point into ngcc (aNGular Compatibility Compiler).
|
||||
*
|
||||
|
|
@ -115,8 +126,18 @@ export function mainNgcc(
|
|||
compileAllFormats = true, createNewEntryPointFormats = false,
|
||||
logger = new ConsoleLogger(LogLevel.info), pathMappings, async = false}: NgccOptions): void|
|
||||
Promise<void> {
|
||||
// Execute in parallel, if async execution is acceptable and there are more than 1 CPU cores.
|
||||
const inParallel = async && (os.cpus().length > 1);
|
||||
|
||||
// Instantiate common utilities that are always used.
|
||||
// NOTE: Avoid eagerly instantiating anything that might not be used when running sync/async or in
|
||||
// master/worker process.
|
||||
const fileSystem = getFileSystem();
|
||||
const pkgJsonUpdater = new DirectPackageJsonUpdater(fileSystem);
|
||||
// NOTE: To avoid file corruption, ensure that each `ngcc` invocation only creates _one_ instance
|
||||
// of `PackageJsonUpdater` that actually writes to disk (across all processes).
|
||||
// This is hard to enforce automatically, when running on multiple processes, so needs to be
|
||||
// enforced manually.
|
||||
const pkgJsonUpdater = getPackageJsonUpdater(inParallel, fileSystem);
|
||||
|
||||
// The function for performing the analysis.
|
||||
const analyzeEntryPoints: AnalyzeEntryPointsFn = () => {
|
||||
|
|
@ -135,7 +156,7 @@ export function mainNgcc(
|
|||
|
||||
const absBasePath = absoluteFrom(basePath);
|
||||
const config = new NgccConfiguration(fileSystem, dirname(absBasePath));
|
||||
const entryPoints = getEntryPoints(
|
||||
const {entryPoints, graph} = getEntryPoints(
|
||||
fileSystem, pkgJsonUpdater, logger, dependencyResolver, config, absBasePath,
|
||||
targetEntryPointPath, pathMappings, supportedPropertiesToConsider, compileAllFormats);
|
||||
|
||||
|
|
@ -176,7 +197,7 @@ export function mainNgcc(
|
|||
unprocessableEntryPointPaths.map(path => `\n - ${path}`).join(''));
|
||||
}
|
||||
|
||||
return new SerialTaskQueue(tasks);
|
||||
return getTaskQueue(inParallel, tasks, graph);
|
||||
};
|
||||
|
||||
// The function for creating the `compile()` function.
|
||||
|
|
@ -233,7 +254,7 @@ export function mainNgcc(
|
|||
};
|
||||
|
||||
// The executor for actually planning and getting the work done.
|
||||
const executor = getExecutor(async, logger, pkgJsonUpdater);
|
||||
const executor = getExecutor(async, inParallel, logger, pkgJsonUpdater);
|
||||
|
||||
return executor.execute(analyzeEntryPoints, createCompileFn);
|
||||
}
|
||||
|
|
@ -260,6 +281,11 @@ function ensureSupportedProperties(properties: string[]): EntryPointJsonProperty
|
|||
return supportedProperties;
|
||||
}
|
||||
|
||||
function getPackageJsonUpdater(inParallel: boolean, fs: FileSystem): PackageJsonUpdater {
|
||||
const directPkgJsonUpdater = new DirectPackageJsonUpdater(fs);
|
||||
return inParallel ? new ClusterPackageJsonUpdater(directPkgJsonUpdater) : directPkgJsonUpdater;
|
||||
}
|
||||
|
||||
function getFileWriter(
|
||||
fs: FileSystem, pkgJsonUpdater: PackageJsonUpdater,
|
||||
createNewEntryPointFormats: boolean): FileWriter {
|
||||
|
|
@ -267,11 +293,23 @@ function getFileWriter(
|
|||
new InPlaceFileWriter(fs);
|
||||
}
|
||||
|
||||
function getExecutor(async: boolean, logger: Logger, pkgJsonUpdater: PackageJsonUpdater): Executor {
|
||||
if (async) {
|
||||
return new AsyncSingleProcessExecutor(logger, pkgJsonUpdater);
|
||||
function getTaskQueue(
|
||||
inParallel: boolean, tasks: PartiallyOrderedTasks, graph: DepGraph<EntryPoint>): TaskQueue {
|
||||
return inParallel ? new ParallelTaskQueue(tasks, graph) : new SerialTaskQueue(tasks);
|
||||
}
|
||||
|
||||
function getExecutor(
|
||||
async: boolean, inParallel: boolean, logger: Logger,
|
||||
pkgJsonUpdater: PackageJsonUpdater): Executor {
|
||||
if (inParallel) {
|
||||
// Execute in parallel (which implies async).
|
||||
// Use up to 8 CPU cores for workers, always reserving one for master.
|
||||
const workerCount = Math.min(8, os.cpus().length - 1);
|
||||
return new ClusterExecutor(workerCount, logger, pkgJsonUpdater);
|
||||
} else {
|
||||
return new SingleProcessExecutor(logger, pkgJsonUpdater);
|
||||
// Execute serially, on a single thread (either sync or async).
|
||||
return async ? new AsyncSingleProcessExecutor(logger, pkgJsonUpdater) :
|
||||
new SingleProcessExecutor(logger, pkgJsonUpdater);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -279,14 +317,15 @@ function getEntryPoints(
|
|||
fs: FileSystem, pkgJsonUpdater: PackageJsonUpdater, logger: Logger,
|
||||
resolver: DependencyResolver, config: NgccConfiguration, basePath: AbsoluteFsPath,
|
||||
targetEntryPointPath: string | undefined, pathMappings: PathMappings | undefined,
|
||||
propertiesToConsider: string[], compileAllFormats: boolean): PartiallyOrderedEntryPoints {
|
||||
const {entryPoints, invalidEntryPoints} = (targetEntryPointPath !== undefined) ?
|
||||
propertiesToConsider: string[], compileAllFormats: boolean):
|
||||
{entryPoints: PartiallyOrderedEntryPoints, graph: DepGraph<EntryPoint>} {
|
||||
const {entryPoints, invalidEntryPoints, graph} = (targetEntryPointPath !== undefined) ?
|
||||
getTargetedEntryPoints(
|
||||
fs, pkgJsonUpdater, logger, resolver, config, basePath, targetEntryPointPath,
|
||||
propertiesToConsider, compileAllFormats, pathMappings) :
|
||||
getAllEntryPoints(fs, config, logger, resolver, basePath, pathMappings);
|
||||
logInvalidEntryPoints(logger, invalidEntryPoints);
|
||||
return entryPoints;
|
||||
return {entryPoints, graph};
|
||||
}
|
||||
|
||||
function getTargetedEntryPoints(
|
||||
|
|
@ -302,6 +341,7 @@ function getTargetedEntryPoints(
|
|||
entryPoints: [] as unknown as PartiallyOrderedEntryPoints,
|
||||
invalidEntryPoints: [],
|
||||
ignoredDependencies: [],
|
||||
graph: EMPTY_GRAPH,
|
||||
};
|
||||
}
|
||||
const finder = new TargetedEntryPointFinder(
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ export type EntryPointFormat = 'esm5' | 'esm2015' | 'umd' | 'commonjs';
|
|||
* An object containing information about an entry-point, including paths
|
||||
* to each of the possible entry-point formats.
|
||||
*/
|
||||
export interface EntryPoint {
|
||||
export interface EntryPoint extends JsonObject {
|
||||
/** The name of the package (e.g. `@angular/core`). */
|
||||
name: string;
|
||||
/** The parsed package.json file for this entry-point. */
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@
|
|||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {DepGraph} from 'dependency-graph';
|
||||
|
||||
import {FileSystem, absoluteFrom, getFileSystem} from '../../../src/ngtsc/file_system';
|
||||
import {runInEachFileSystem} from '../../../src/ngtsc/file_system/testing';
|
||||
import {DependencyResolver, SortedEntryPointsInfo} from '../../src/dependencies/dependency_resolver';
|
||||
|
|
@ -13,6 +16,7 @@ import {ModuleResolver} from '../../src/dependencies/module_resolver';
|
|||
import {EntryPoint} from '../../src/packages/entry_point';
|
||||
import {MockLogger} from '../helpers/mock_logger';
|
||||
|
||||
|
||||
interface DepMap {
|
||||
[path: string]: {resolved: string[], missing: string[]};
|
||||
}
|
||||
|
|
@ -162,6 +166,15 @@ runInEachFileSystem(() => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('should return the computed dependency graph', () => {
|
||||
spyOn(host, 'findDependencies').and.callFake(createFakeComputeDependencies(dependencies));
|
||||
const result = resolver.sortEntryPointsByDependency([fifth, first, fourth, second, third]);
|
||||
|
||||
expect(result.graph).toEqual(jasmine.any(DepGraph));
|
||||
expect(result.graph.size()).toBe(5);
|
||||
expect(result.graph.dependenciesOf(third.path)).toEqual([fifth.path, fourth.path]);
|
||||
});
|
||||
|
||||
it('should only return dependencies of the target, if provided', () => {
|
||||
spyOn(host, 'findDependencies').and.callFake(createFakeComputeDependencies(dependencies));
|
||||
const entryPoints = [fifth, first, fourth, second, third];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/// <reference types="node" />
|
||||
|
||||
import * as cluster from 'cluster';
|
||||
|
||||
import {ClusterExecutor} from '../../../src/execution/cluster/executor';
|
||||
import {ClusterMaster} from '../../../src/execution/cluster/master';
|
||||
import {ClusterWorker} from '../../../src/execution/cluster/worker';
|
||||
import {PackageJsonUpdater} from '../../../src/writing/package_json_updater';
|
||||
import {MockLogger} from '../../helpers/mock_logger';
|
||||
import {mockProperty} from '../../helpers/spy_utils';
|
||||
|
||||
|
||||
describe('ClusterExecutor', () => {
|
||||
const runAsClusterMaster = mockProperty(cluster, 'isMaster');
|
||||
let masterRunSpy: jasmine.Spy;
|
||||
let workerRunSpy: jasmine.Spy;
|
||||
let mockLogger: MockLogger;
|
||||
let executor: ClusterExecutor;
|
||||
|
||||
beforeEach(() => {
|
||||
masterRunSpy = spyOn(ClusterMaster.prototype, 'run');
|
||||
workerRunSpy = spyOn(ClusterWorker.prototype, 'run');
|
||||
|
||||
mockLogger = new MockLogger();
|
||||
executor = new ClusterExecutor(42, mockLogger, null as unknown as PackageJsonUpdater);
|
||||
});
|
||||
|
||||
describe('execute()', () => {
|
||||
describe('(on cluster master)', () => {
|
||||
beforeEach(() => runAsClusterMaster(true));
|
||||
|
||||
it('should log debug info about the executor', () => {
|
||||
const anyFn: () => any = () => undefined;
|
||||
executor.execute(anyFn, anyFn);
|
||||
|
||||
expect(mockLogger.logs.debug).toEqual([
|
||||
['Running ngcc on ClusterExecutor (using 42 worker processes).'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('should delegate to `ClusterMaster#run()`', async() => {
|
||||
masterRunSpy.and.returnValue('CusterMaster#run()');
|
||||
const analyzeEntryPointsSpy = jasmine.createSpy('analyzeEntryPoints');
|
||||
const createCompilerFnSpy = jasmine.createSpy('createCompilerFn');
|
||||
|
||||
expect(await executor.execute(analyzeEntryPointsSpy, createCompilerFnSpy))
|
||||
.toBe('CusterMaster#run()' as any);
|
||||
|
||||
expect(masterRunSpy).toHaveBeenCalledWith();
|
||||
expect(workerRunSpy).not.toHaveBeenCalled();
|
||||
|
||||
expect(analyzeEntryPointsSpy).toHaveBeenCalledWith();
|
||||
expect(createCompilerFnSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('(on cluster worker)', () => {
|
||||
beforeEach(() => runAsClusterMaster(false));
|
||||
|
||||
it('should not log debug info about the executor', () => {
|
||||
const anyFn: () => any = () => undefined;
|
||||
executor.execute(anyFn, anyFn);
|
||||
|
||||
expect(mockLogger.logs.debug).toEqual([]);
|
||||
});
|
||||
|
||||
it('should delegate to `ClusterWorker#run()`', async() => {
|
||||
workerRunSpy.and.returnValue('CusterWorker#run()');
|
||||
const analyzeEntryPointsSpy = jasmine.createSpy('analyzeEntryPoints');
|
||||
const createCompilerFnSpy = jasmine.createSpy('createCompilerFn');
|
||||
|
||||
expect(await executor.execute(analyzeEntryPointsSpy, createCompilerFnSpy))
|
||||
.toBe('CusterWorker#run()' as any);
|
||||
|
||||
expect(masterRunSpy).not.toHaveBeenCalledWith();
|
||||
expect(workerRunSpy).toHaveBeenCalled();
|
||||
|
||||
expect(analyzeEntryPointsSpy).not.toHaveBeenCalled();
|
||||
expect(createCompilerFnSpy).toHaveBeenCalledWith(jasmine.any(Function));
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,198 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/// <reference types="node" />
|
||||
|
||||
import * as cluster from 'cluster';
|
||||
|
||||
import {absoluteFrom as _} from '../../../../src/ngtsc/file_system';
|
||||
import {runInEachFileSystem} from '../../../../src/ngtsc/file_system/testing';
|
||||
import {ClusterPackageJsonUpdater} from '../../../src/execution/cluster/package_json_updater';
|
||||
import {JsonObject} from '../../../src/packages/entry_point';
|
||||
import {PackageJsonUpdate, PackageJsonUpdater} from '../../../src/writing/package_json_updater';
|
||||
import {mockProperty} from '../../helpers/spy_utils';
|
||||
|
||||
|
||||
runInEachFileSystem(() => {
|
||||
describe('ClusterPackageJsonUpdater', () => {
|
||||
const runAsClusterMaster = mockProperty(cluster, 'isMaster');
|
||||
const mockProcessSend = mockProperty(process, 'send');
|
||||
let processSendSpy: jasmine.Spy;
|
||||
let delegate: PackageJsonUpdater;
|
||||
let updater: ClusterPackageJsonUpdater;
|
||||
|
||||
beforeEach(() => {
|
||||
processSendSpy = jasmine.createSpy('process.send');
|
||||
mockProcessSend(processSendSpy);
|
||||
|
||||
delegate = new MockPackageJsonUpdater();
|
||||
updater = new ClusterPackageJsonUpdater(delegate);
|
||||
});
|
||||
|
||||
describe('createUpdate()', () => {
|
||||
[true, false].forEach(
|
||||
isMaster => describe(`(on cluster ${isMaster ? 'master' : 'worker'})`, () => {
|
||||
beforeEach(() => runAsClusterMaster(isMaster));
|
||||
|
||||
it('should return a `PackageJsonUpdate` instance',
|
||||
() => { expect(updater.createUpdate()).toEqual(jasmine.any(PackageJsonUpdate)); });
|
||||
|
||||
it('should wire up the `PackageJsonUpdate` with its `writeChanges()` method', () => {
|
||||
const writeChangesSpy = spyOn(updater, 'writeChanges');
|
||||
const jsonPath = _('/foo/package.json');
|
||||
const update = updater.createUpdate();
|
||||
|
||||
update.addChange(['foo'], 'updated');
|
||||
update.writeChanges(jsonPath);
|
||||
|
||||
expect(writeChangesSpy)
|
||||
.toHaveBeenCalledWith([[['foo'], 'updated']], jsonPath, undefined);
|
||||
});
|
||||
}));
|
||||
});
|
||||
|
||||
describe('writeChanges()', () => {
|
||||
describe('(on cluster master)', () => {
|
||||
beforeEach(() => runAsClusterMaster(true));
|
||||
afterEach(() => expect(processSendSpy).not.toHaveBeenCalled());
|
||||
|
||||
it('should forward the call to the delegate `PackageJsonUpdater`', () => {
|
||||
const jsonPath = _('/foo/package.json');
|
||||
const parsedJson = {foo: 'bar'};
|
||||
|
||||
updater.createUpdate().addChange(['foo'], 'updated').writeChanges(jsonPath, parsedJson);
|
||||
|
||||
expect(delegate.writeChanges)
|
||||
.toHaveBeenCalledWith([[['foo'], 'updated']], jsonPath, parsedJson);
|
||||
});
|
||||
|
||||
it('should throw, if trying to re-apply an already applied update', () => {
|
||||
const update = updater.createUpdate().addChange(['foo'], 'updated');
|
||||
|
||||
expect(() => update.writeChanges(_('/foo/package.json'))).not.toThrow();
|
||||
expect(() => update.writeChanges(_('/foo/package.json')))
|
||||
.toThrowError('Trying to apply a `PackageJsonUpdate` that has already been applied.');
|
||||
expect(() => update.writeChanges(_('/bar/package.json')))
|
||||
.toThrowError('Trying to apply a `PackageJsonUpdate` that has already been applied.');
|
||||
});
|
||||
});
|
||||
|
||||
describe('(on cluster worker)', () => {
|
||||
beforeEach(() => runAsClusterMaster(false));
|
||||
|
||||
it('should send an `update-package-json` message to the master process', () => {
|
||||
const jsonPath = _('/foo/package.json');
|
||||
|
||||
const writeToProp = (propPath: string[], parsed?: JsonObject) =>
|
||||
updater.createUpdate().addChange(propPath, 'updated').writeChanges(jsonPath, parsed);
|
||||
|
||||
writeToProp(['foo']);
|
||||
expect(processSendSpy).toHaveBeenCalledWith({
|
||||
type: 'update-package-json',
|
||||
packageJsonPath: jsonPath,
|
||||
changes: [[['foo'], 'updated']],
|
||||
});
|
||||
|
||||
writeToProp(['bar', 'baz', 'qux'], {});
|
||||
expect(processSendSpy).toHaveBeenCalledWith({
|
||||
type: 'update-package-json',
|
||||
packageJsonPath: jsonPath,
|
||||
changes: [[['bar', 'baz', 'qux'], 'updated']],
|
||||
});
|
||||
});
|
||||
|
||||
it('should update an in-memory representation (if provided)', () => {
|
||||
const jsonPath = _('/foo/package.json');
|
||||
const parsedJson: JsonObject = {
|
||||
foo: true,
|
||||
bar: {baz: 'OK'},
|
||||
};
|
||||
|
||||
const update =
|
||||
updater.createUpdate().addChange(['foo'], false).addChange(['bar', 'baz'], 42);
|
||||
|
||||
// Not updated yet.
|
||||
expect(parsedJson).toEqual({
|
||||
foo: true,
|
||||
bar: {baz: 'OK'},
|
||||
});
|
||||
|
||||
update.writeChanges(jsonPath, parsedJson);
|
||||
|
||||
// Updated now.
|
||||
expect(parsedJson).toEqual({
|
||||
foo: false,
|
||||
bar: {baz: 42},
|
||||
});
|
||||
});
|
||||
|
||||
it('should create any missing ancestor objects', () => {
|
||||
const jsonPath = _('/foo/package.json');
|
||||
const parsedJson: JsonObject = {foo: {}};
|
||||
|
||||
updater.createUpdate()
|
||||
.addChange(['foo', 'bar', 'baz', 'qux'], 'updated')
|
||||
.writeChanges(jsonPath, parsedJson);
|
||||
|
||||
expect(parsedJson).toEqual({
|
||||
foo: {
|
||||
bar: {
|
||||
baz: {
|
||||
qux: 'updated',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should throw, if a property-path is empty', () => {
|
||||
const jsonPath = _('/foo/package.json');
|
||||
|
||||
expect(() => updater.createUpdate().addChange([], 'missing').writeChanges(jsonPath, {}))
|
||||
.toThrowError(`Missing property path for writing value to '${jsonPath}'.`);
|
||||
});
|
||||
|
||||
it('should throw, if a property-path points to a non-object intermediate value', () => {
|
||||
const jsonPath = _('/foo/package.json');
|
||||
const parsedJson = {foo: null, bar: 42, baz: {qux: []}};
|
||||
|
||||
const writeToProp = (propPath: string[], parsed?: JsonObject) =>
|
||||
updater.createUpdate().addChange(propPath, 'updated').writeChanges(jsonPath, parsed);
|
||||
|
||||
expect(() => writeToProp(['foo', 'child'], parsedJson))
|
||||
.toThrowError('Property path \'foo.child\' does not point to an object.');
|
||||
expect(() => writeToProp(['bar', 'child'], parsedJson))
|
||||
.toThrowError('Property path \'bar.child\' does not point to an object.');
|
||||
expect(() => writeToProp(['baz', 'qux', 'child'], parsedJson))
|
||||
.toThrowError('Property path \'baz.qux.child\' does not point to an object.');
|
||||
|
||||
// It should not throw, if no parsed representation is provided.
|
||||
// (The error will still be thrown on the master process, but that is out of scope for
|
||||
// this test.)
|
||||
expect(() => writeToProp(['foo', 'child'])).not.toThrow();
|
||||
});
|
||||
|
||||
it('should throw, if trying to re-apply an already applied update', () => {
|
||||
const update = updater.createUpdate().addChange(['foo'], 'updated');
|
||||
|
||||
expect(() => update.writeChanges(_('/foo/package.json'))).not.toThrow();
|
||||
expect(() => update.writeChanges(_('/foo/package.json')))
|
||||
.toThrowError('Trying to apply a `PackageJsonUpdate` that has already been applied.');
|
||||
expect(() => update.writeChanges(_('/bar/package.json')))
|
||||
.toThrowError('Trying to apply a `PackageJsonUpdate` that has already been applied.');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Helpers
|
||||
class MockPackageJsonUpdater implements PackageJsonUpdater {
|
||||
createUpdate = jasmine.createSpy('MockPackageJsonUpdater#createUpdate');
|
||||
writeChanges = jasmine.createSpy('MockPackageJsonUpdater#writeChanges');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,144 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/// <reference types="node" />
|
||||
|
||||
import * as cluster from 'cluster';
|
||||
import {EventEmitter} from 'events';
|
||||
|
||||
import {Task, TaskCompletedCallback, TaskProcessingOutcome} from '../../../src/execution/api';
|
||||
import {ClusterWorker} from '../../../src/execution/cluster/worker';
|
||||
import {mockProperty} from '../../helpers/spy_utils';
|
||||
|
||||
|
||||
describe('ClusterWorker', () => {
|
||||
const runAsClusterMaster = mockProperty(cluster, 'isMaster');
|
||||
const mockProcessSend = mockProperty(process, 'send');
|
||||
let processSendSpy: jasmine.Spy;
|
||||
let compileFnSpy: jasmine.Spy;
|
||||
let createCompileFnSpy: jasmine.Spy;
|
||||
|
||||
beforeEach(() => {
|
||||
compileFnSpy = jasmine.createSpy('compileFn');
|
||||
createCompileFnSpy = jasmine.createSpy('createCompileFn').and.returnValue(compileFnSpy);
|
||||
|
||||
processSendSpy = jasmine.createSpy('process.send');
|
||||
mockProcessSend(processSendSpy);
|
||||
});
|
||||
|
||||
describe('constructor()', () => {
|
||||
describe('(on cluster master)', () => {
|
||||
beforeEach(() => runAsClusterMaster(true));
|
||||
|
||||
it('should throw an error', () => {
|
||||
expect(() => new ClusterWorker(createCompileFnSpy))
|
||||
.toThrowError('Tried to instantiate `ClusterWorker` on the master process.');
|
||||
expect(createCompileFnSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('(on cluster worker)', () => {
|
||||
beforeEach(() => runAsClusterMaster(false));
|
||||
|
||||
it('should create the `compileFn()`', () => {
|
||||
new ClusterWorker(createCompileFnSpy);
|
||||
expect(createCompileFnSpy).toHaveBeenCalledWith(jasmine.any(Function));
|
||||
});
|
||||
|
||||
it('should set up `compileFn()` to send a `task-completed` message to master', () => {
|
||||
new ClusterWorker(createCompileFnSpy);
|
||||
const onTaskCompleted: TaskCompletedCallback = createCompileFnSpy.calls.argsFor(0)[0];
|
||||
|
||||
onTaskCompleted(null as any, TaskProcessingOutcome.AlreadyProcessed);
|
||||
expect(processSendSpy).toHaveBeenCalledTimes(1);
|
||||
expect(processSendSpy).toHaveBeenCalledWith({
|
||||
type: 'task-completed',
|
||||
outcome: TaskProcessingOutcome.AlreadyProcessed,
|
||||
});
|
||||
|
||||
onTaskCompleted(null as any, TaskProcessingOutcome.Processed);
|
||||
expect(processSendSpy).toHaveBeenCalledTimes(2);
|
||||
expect(processSendSpy).toHaveBeenCalledWith({
|
||||
type: 'task-completed',
|
||||
outcome: TaskProcessingOutcome.Processed,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('run()', () => {
|
||||
describe(
|
||||
'(on cluster master)',
|
||||
() => {/* No tests needed, becasue the constructor would have thrown. */});
|
||||
|
||||
describe('(on cluster worker)', () => {
|
||||
// The `cluster.worker` property is normally `undefined` on the master process and set to the
|
||||
// current `cluster.Worker` on worker processes.
|
||||
const mockClusterWorker = mockProperty(cluster, 'worker');
|
||||
let worker: ClusterWorker;
|
||||
|
||||
beforeEach(() => {
|
||||
runAsClusterMaster(false);
|
||||
mockClusterWorker(Object.assign(new EventEmitter(), {id: 42}) as cluster.Worker);
|
||||
|
||||
worker = new ClusterWorker(createCompileFnSpy);
|
||||
});
|
||||
|
||||
it('should return a promise (that is never resolved)', done => {
|
||||
const promise = worker.run();
|
||||
|
||||
expect(promise).toEqual(jasmine.any(Promise));
|
||||
|
||||
promise.then(
|
||||
() => done.fail('Expected promise not to resolve'),
|
||||
() => done.fail('Expected promise not to reject'));
|
||||
|
||||
// We can't wait forever to verify that the promise is not resolved, but at least verify
|
||||
// that it is not resolved immediately.
|
||||
setTimeout(done, 100);
|
||||
});
|
||||
|
||||
it('should handle `process-task` messages', () => {
|
||||
const mockTask = { foo: 'bar' } as unknown as Task;
|
||||
|
||||
worker.run();
|
||||
cluster.worker.emit('message', {type: 'process-task', task: mockTask});
|
||||
|
||||
expect(compileFnSpy).toHaveBeenCalledWith(mockTask);
|
||||
expect(processSendSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should send errors during task processing back to the master process', () => {
|
||||
let err: string|Error;
|
||||
compileFnSpy.and.callFake(() => { throw err; });
|
||||
|
||||
worker.run();
|
||||
|
||||
err = 'Error string.';
|
||||
cluster.worker.emit('message', {type: 'process-task', task: {} as Task});
|
||||
expect(processSendSpy).toHaveBeenCalledWith({type: 'error', error: err});
|
||||
|
||||
err = new Error('Error object.');
|
||||
cluster.worker.emit('message', {type: 'process-task', task: {} as Task});
|
||||
expect(processSendSpy).toHaveBeenCalledWith({type: 'error', error: err.stack});
|
||||
});
|
||||
|
||||
it('should throw, when an unknown message type is received', () => {
|
||||
worker.run();
|
||||
cluster.worker.emit('message', {type: 'unknown', foo: 'bar'});
|
||||
|
||||
expect(compileFnSpy).not.toHaveBeenCalled();
|
||||
expect(processSendSpy).toHaveBeenCalledWith({
|
||||
type: 'error',
|
||||
error: jasmine.stringMatching(
|
||||
'Error: Invalid message received on worker #42: {"type":"unknown","foo":"bar"}'),
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/** An object with helpers for mocking/spying on an object's property. */
|
||||
export interface IPropertySpyHelpers<T, P extends keyof T> {
|
||||
/**
|
||||
* A `jasmine.Spy` for `get` operations on the property (i.e. reading the current property value).
|
||||
* (This is useful in case one needs to make assertions against property reads.)
|
||||
*/
|
||||
getSpy: jasmine.Spy;
|
||||
|
||||
/**
|
||||
* A `jasmine.Spy` for `set` operations on the property (i.e. setting a new property value).
|
||||
* (This is useful in case one needs to make assertions against property writes.)
|
||||
*/
|
||||
setSpy: jasmine.Spy;
|
||||
|
||||
/** Install the getter/setter spies for the property. */
|
||||
installSpies(): void;
|
||||
|
||||
/**
|
||||
* Uninstall the property spies and restore the original value (from before installing the
|
||||
* spies), including the property descriptor.
|
||||
*/
|
||||
uninstallSpies(): void;
|
||||
|
||||
/** Update the current value of the mocked property. */
|
||||
setMockValue(value: T[P]): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up mocking an object's property (using spies) and return a function for updating the mocked
|
||||
* property's value during tests.
|
||||
*
|
||||
* This is, essentially, a wrapper around `spyProperty()` which additionally takes care of
|
||||
* installing the spies before each test (via `beforeEach()`) and uninstalling them after each test
|
||||
* (via `afterEach()`).
|
||||
*
|
||||
* Example usage:
|
||||
*
|
||||
* ```ts
|
||||
* describe('something', () => {
|
||||
* // Assuming `window.foo` is an object...
|
||||
* const mockWindowFooBar = mockProperty(window.foo, 'bar');
|
||||
*
|
||||
* it('should do this', () => {
|
||||
* mockWindowFooBar('baz');
|
||||
* expect(window.foo.bar).toBe('baz');
|
||||
*
|
||||
* mockWindowFooBar('qux');
|
||||
* expect(window.foo.bar).toBe('qux');
|
||||
* });
|
||||
* });
|
||||
* ```
|
||||
*
|
||||
* @param ctx The object whose property needs to be spied on.
|
||||
* @param prop The name of the property to spy on.
|
||||
*
|
||||
* @return A function for updating the current value of the mocked property.
|
||||
*/
|
||||
export const mockProperty =
|
||||
<T, P extends keyof T>(ctx: T, prop: P): IPropertySpyHelpers<T, P>['setMockValue'] => {
|
||||
const {setMockValue, installSpies, uninstallSpies} = spyProperty(ctx, prop);
|
||||
|
||||
beforeEach(installSpies);
|
||||
afterEach(uninstallSpies);
|
||||
|
||||
return setMockValue;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return utility functions to help mock and spy on an object's property.
|
||||
*
|
||||
* It supports spying on properties that are either defined on the object instance itself or on its
|
||||
* prototype. It also supports spying on non-writable properties (as long as they are configurable).
|
||||
*
|
||||
* NOTE: Unlike `jasmine`'s spying utilities, spies are not automatically installed/uninstalled, so
|
||||
* the caller is responsible for manually taking care of that (by calling
|
||||
* `installSpies()`/`uninstallSpies()` as necessary).
|
||||
*
|
||||
* @param ctx The object whose property needs to be spied on.
|
||||
* @param prop The name of the property to spy on.
|
||||
*
|
||||
* @return An object with helpers for mocking/spying on an object's property.
|
||||
*/
|
||||
export const spyProperty = <T, P extends keyof T>(ctx: T, prop: P): IPropertySpyHelpers<T, P> => {
|
||||
const originalDescriptor = Object.getOwnPropertyDescriptor(ctx, prop);
|
||||
|
||||
let value = ctx[prop];
|
||||
const setMockValue = (mockValue: typeof value) => value = mockValue;
|
||||
const setSpy = jasmine.createSpy(`set ${prop}`).and.callFake(setMockValue);
|
||||
const getSpy = jasmine.createSpy(`get ${prop}`).and.callFake(() => value);
|
||||
|
||||
const installSpies = () => {
|
||||
value = ctx[prop];
|
||||
Object.defineProperty(ctx, prop, {
|
||||
configurable: true,
|
||||
enumerable: originalDescriptor ? originalDescriptor.enumerable : true,
|
||||
get: getSpy,
|
||||
set: setSpy,
|
||||
});
|
||||
};
|
||||
const uninstallSpies = () =>
|
||||
originalDescriptor ? Object.defineProperty(ctx, prop, originalDescriptor) : delete ctx[prop];
|
||||
|
||||
return {installSpies, uninstallSpies, setMockValue, getSpy, setSpy};
|
||||
};
|
||||
|
|
@ -5,6 +5,11 @@
|
|||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/// <reference types="node" />
|
||||
|
||||
import * as os from 'os';
|
||||
|
||||
import {AbsoluteFsPath, FileSystem, absoluteFrom, getFileSystem, join} from '../../../src/ngtsc/file_system';
|
||||
import {Folder, MockFileSystem, TestFile, runInEachFileSystem} from '../../../src/ngtsc/file_system/testing';
|
||||
import {loadStandardTestFiles, loadTestFiles} from '../../../test/helpers';
|
||||
|
|
@ -15,6 +20,7 @@ import {Transformer} from '../../src/packages/transformer';
|
|||
import {DirectPackageJsonUpdater, PackageJsonUpdater} from '../../src/writing/package_json_updater';
|
||||
import {MockLogger} from '../helpers/mock_logger';
|
||||
|
||||
|
||||
const testFiles = loadStandardTestFiles({fakeCore: false, rxjs: true});
|
||||
|
||||
runInEachFileSystem(() => {
|
||||
|
|
@ -28,6 +34,9 @@ runInEachFileSystem(() => {
|
|||
fs = getFileSystem();
|
||||
pkgJsonUpdater = new DirectPackageJsonUpdater(fs);
|
||||
initMockFileSystem(fs, testFiles);
|
||||
|
||||
// Force single-process execution in unit tests by mocking available CPUs to 1.
|
||||
spyOn(os, 'cpus').and.returnValue([{model: 'Mock CPU'}]);
|
||||
});
|
||||
|
||||
it('should run ngcc without errors for esm2015', () => {
|
||||
|
|
@ -402,7 +411,6 @@ runInEachFileSystem(() => {
|
|||
propertiesToConsider: ['module', 'fesm5', 'esm5'],
|
||||
compileAllFormats: false,
|
||||
logger: new MockLogger(),
|
||||
|
||||
});
|
||||
// * In the Angular packages fesm5 and module have the same underlying format,
|
||||
// so both are marked as compiled.
|
||||
|
|
|
|||
Loading…
Reference in New Issue