From e367593a26aed9ded40cb0aa6acd2bc182e1ff13 Mon Sep 17 00:00:00 2001 From: George Kalpakas Date: Wed, 29 Apr 2020 21:28:11 +0300 Subject: [PATCH] refactor(ngcc): support running callback before writing transformed files (#36626) This commit enhances the `CompileFn`, which is used to process each entry-point, to support running a passed-in callback (and wait for it to complete) before proceeding with writing the transformed files to disk. This functionality is currently not used. In a subsequent commit, it will be used for passing info from worker processes to the master process that will allow ngcc to recover when a worker process crashes in the middle of processing a task. PR Close #36626 --- packages/compiler-cli/ngcc/src/execution/api.ts | 7 +++++-- .../ngcc/src/execution/cluster/worker.ts | 5 +++-- .../src/execution/create_compile_function.ts | 17 +++++++++++++---- .../src/execution/single_process_executor.ts | 2 +- .../ngcc/test/execution/cluster/worker_spec.ts | 4 ++-- .../execution/single_processor_executor_spec.ts | 7 ++++--- 6 files changed, 28 insertions(+), 14 deletions(-) diff --git a/packages/compiler-cli/ngcc/src/execution/api.ts b/packages/compiler-cli/ngcc/src/execution/api.ts index a6418984b3..1c95403301 100644 --- a/packages/compiler-cli/ngcc/src/execution/api.ts +++ b/packages/compiler-cli/ngcc/src/execution/api.ts @@ -5,6 +5,7 @@ * Use of this source code is governed by an MIT-style license that can be * found in the LICENSE file at https://angular.io/license */ +import {FileToWrite} from '../rendering/utils'; import {Task, TaskCompletedCallback, TaskQueue} from './tasks/api'; /** @@ -16,10 +17,12 @@ import {Task, TaskCompletedCallback, TaskQueue} from './tasks/api'; export type AnalyzeEntryPointsFn = () => TaskQueue; /** The type of the function that can process/compile a task. */ -export type CompileFn = (task: Task) => void; +export type CompileFn = (task: Task) => void|T; /** The type of the function that creates the `CompileFn` function used to process tasks. */ -export type CreateCompileFn = (onTaskCompleted: TaskCompletedCallback) => CompileFn; +export type CreateCompileFn = >( + beforeWritingFiles: (transformedFiles: FileToWrite[]) => T, + onTaskCompleted: TaskCompletedCallback) => CompileFn; /** * A class that orchestrates and executes the required work (i.e. analyzes the entry-points, diff --git a/packages/compiler-cli/ngcc/src/execution/cluster/worker.ts b/packages/compiler-cli/ngcc/src/execution/cluster/worker.ts index ca0e57aec5..e825a1b69f 100644 --- a/packages/compiler-cli/ngcc/src/execution/cluster/worker.ts +++ b/packages/compiler-cli/ngcc/src/execution/cluster/worker.ts @@ -62,17 +62,18 @@ export async function startWorker(logger: Logger, createCompileFn: CreateCompile } const compile = createCompileFn( + () => {}, (_task, outcome, message) => sendMessageToMaster({type: 'task-completed', outcome, message})); // Listen for `ProcessTaskMessage`s and process tasks. - cluster.worker.on('message', (msg: MessageToWorker) => { + cluster.worker.on('message', async (msg: MessageToWorker) => { try { switch (msg.type) { case 'process-task': logger.debug( `[Worker #${cluster.worker.id}] Processing task: ${stringifyTask(msg.task)}`); - return compile(msg.task); + return await compile(msg.task); default: throw new Error( `[Worker #${cluster.worker.id}] Invalid message received: ${JSON.stringify(msg)}`); diff --git a/packages/compiler-cli/ngcc/src/execution/create_compile_function.ts b/packages/compiler-cli/ngcc/src/execution/create_compile_function.ts index 0998129b4c..5a2e262867 100644 --- a/packages/compiler-cli/ngcc/src/execution/create_compile_function.ts +++ b/packages/compiler-cli/ngcc/src/execution/create_compile_function.ts @@ -31,7 +31,7 @@ export function getCreateCompileFn( createNewEntryPointFormats: boolean, errorOnFailedEntryPoint: boolean, enableI18nLegacyMessageIdFormat: boolean, tsConfig: ParsedConfiguration|null, pathMappings: PathMappings|undefined): CreateCompileFn { - return onTaskCompleted => { + return (beforeWritingFiles, onTaskCompleted) => { const fileWriter = getFileWriter( fileSystem, logger, pkgJsonUpdater, createNewEntryPointFormats, errorOnFailedEntryPoint); const {Transformer} = require('../packages/transformer'); @@ -69,11 +69,20 @@ export function getCreateCompileFn( logger.warn(replaceTsWithNgInErrors( ts.formatDiagnosticsWithColorAndContext(result.diagnostics, bundle.src.host))); } - fileWriter.writeBundle(bundle, result.transformedFiles, formatPropertiesToMarkAsProcessed); - logger.debug(` Successfully compiled ${entryPoint.name} : ${formatProperty}`); + const writeBundle = () => { + fileWriter.writeBundle( + bundle, result.transformedFiles, formatPropertiesToMarkAsProcessed); - onTaskCompleted(task, TaskProcessingOutcome.Processed, null); + logger.debug(` Successfully compiled ${entryPoint.name} : ${formatProperty}`); + onTaskCompleted(task, TaskProcessingOutcome.Processed, null); + }; + + const beforeWritingResult = beforeWritingFiles(result.transformedFiles); + + return (beforeWritingResult instanceof Promise) ? + beforeWritingResult.then(writeBundle) as ReturnType: + writeBundle(); } else { const errors = replaceTsWithNgInErrors( ts.formatDiagnosticsWithColorAndContext(result.diagnostics, bundle.src.host)); diff --git a/packages/compiler-cli/ngcc/src/execution/single_process_executor.ts b/packages/compiler-cli/ngcc/src/execution/single_process_executor.ts index 982bd7a58d..e487e5d5f3 100644 --- a/packages/compiler-cli/ngcc/src/execution/single_process_executor.ts +++ b/packages/compiler-cli/ngcc/src/execution/single_process_executor.ts @@ -23,7 +23,7 @@ export abstract class SingleProcessorExecutorBase { const taskQueue = analyzeEntryPoints(); const onTaskCompleted = this.createTaskCompletedCallback(taskQueue); - const compile = createCompileFn(onTaskCompleted); + const compile = createCompileFn(() => {}, onTaskCompleted); // Process all tasks. this.logger.debug('Processing tasks...'); diff --git a/packages/compiler-cli/ngcc/test/execution/cluster/worker_spec.ts b/packages/compiler-cli/ngcc/test/execution/cluster/worker_spec.ts index 8cc9370b3c..c0887a6fcc 100644 --- a/packages/compiler-cli/ngcc/test/execution/cluster/worker_spec.ts +++ b/packages/compiler-cli/ngcc/test/execution/cluster/worker_spec.ts @@ -57,12 +57,12 @@ describe('startWorker()', () => { it('should create the `compileFn()`', () => { startWorker(mockLogger, createCompileFnSpy); - expect(createCompileFnSpy).toHaveBeenCalledWith(jasmine.any(Function)); + expect(createCompileFnSpy).toHaveBeenCalledWith(jasmine.any(Function), jasmine.any(Function)); }); it('should set up `compileFn()` to send `task-completed` messages to master', () => { startWorker(mockLogger, createCompileFnSpy); - const onTaskCompleted: TaskCompletedCallback = createCompileFnSpy.calls.argsFor(0)[0]; + const onTaskCompleted: TaskCompletedCallback = createCompileFnSpy.calls.argsFor(0)[1]; onTaskCompleted(null as any, TaskProcessingOutcome.Processed, null); expect(processSendSpy).toHaveBeenCalledTimes(1); diff --git a/packages/compiler-cli/ngcc/test/execution/single_processor_executor_spec.ts b/packages/compiler-cli/ngcc/test/execution/single_processor_executor_spec.ts index a3d38fef09..c4f11a0e17 100644 --- a/packages/compiler-cli/ngcc/test/execution/single_processor_executor_spec.ts +++ b/packages/compiler-cli/ngcc/test/execution/single_processor_executor_spec.ts @@ -132,14 +132,15 @@ describe('SingleProcessExecutor', () => { {allTasksCompleted: true, getNextTask: jasmine.any(Function)})]); }); - it('should pass the created TaskCompletedCallback to the createCompileFn', () => { + it('should pass the necessary callbacks to createCompileFn', () => { + const beforeWritingFiles = jasmine.any(Function); + const onTaskCompleted = () => {}; const createCompileFn = jasmine.createSpy('createCompileFn').and.returnValue(function compileFn() {}); - function onTaskCompleted() {} createTaskCompletedCallback.and.returnValue(onTaskCompleted); executor.execute(noTasks, createCompileFn); expect(createCompileFn).toHaveBeenCalledTimes(1); - expect(createCompileFn).toHaveBeenCalledWith(onTaskCompleted); + expect(createCompileFn).toHaveBeenCalledWith(beforeWritingFiles, onTaskCompleted); }); }); });