From 4779c4b94ad415cc09102f2a50e9db7997becd7c Mon Sep 17 00:00:00 2001
From: George Kalpakas <kalpakas.g@gmail.com>
Date: Wed, 29 Apr 2020 21:28:27 +0300
Subject: [PATCH] fix(ngcc): handle `ENOMEM` errors in worker processes
 (#36626)

When running in parallel mode, worker processes forward errors thrown
during task processing to the master process, which in turn exits with
an error.

However, there are cases where the error is not directly related to
processing the entry-point. One such case is when there is not enough
memory (for example, due to all the other tasks being processed
simultaneously).

Previously, an `ENOMEM` error thrown on a worker process would propagate
to the master process, eventually causing ngcc to exit with an error.
Example failure: https://circleci.com/gh/angular/angular/682198

This commit improves handling of these low-memory situations by
detecting `ENOMEM` errors and killing the worker process, thus allowing
the master process to decide how to handle that. The master process will
put the task back into the tasks queue and continue processing tasks
with the rest of the worker processes (and thus with lower memory
pressure).

PR Close #36626
---
 .../ngcc/src/execution/cluster/worker.ts      | 19 ++++++++---
 .../test/execution/cluster/worker_spec.ts     | 34 ++++++++++++++++++-
 2 files changed, 48 insertions(+), 5 deletions(-)

diff --git a/packages/compiler-cli/ngcc/src/execution/cluster/worker.ts b/packages/compiler-cli/ngcc/src/execution/cluster/worker.ts
index 6cf4c752b9..e018384c06 100644
--- a/packages/compiler-cli/ngcc/src/execution/cluster/worker.ts
+++ b/packages/compiler-cli/ngcc/src/execution/cluster/worker.ts
@@ -81,10 +81,21 @@ export async function startWorker(logger: Logger, createCompileFn: CreateCompile
               `[Worker #${cluster.worker.id}] Invalid message received: ${JSON.stringify(msg)}`);
       }
     } catch (err) {
-      await sendMessageToMaster({
-        type: 'error',
-        error: (err instanceof Error) ? (err.stack || err.message) : err,
-      });
+      switch (err && err.code) {
+        case 'ENOMEM':
+          // Not being able to allocate enough memory is not necessarily a problem with processing
+          // the current task. It could just mean that there are too many tasks being processed
+          // simultaneously.
+          //
+          // Exit with an error and let the cluster master decide how to handle this.
+          logger.warn(`[Worker #${cluster.worker.id}] ${err.stack || err.message}`);
+          return process.exit(1);
+        default:
+          await sendMessageToMaster({
+            type: 'error',
+            error: (err instanceof Error) ? (err.stack || err.message) : err,
+          });
+      }
     }
   });
 
diff --git a/packages/compiler-cli/ngcc/test/execution/cluster/worker_spec.ts b/packages/compiler-cli/ngcc/test/execution/cluster/worker_spec.ts
index e02af404fd..277bf7c5e2 100644
--- a/packages/compiler-cli/ngcc/test/execution/cluster/worker_spec.ts
+++ b/packages/compiler-cli/ngcc/test/execution/cluster/worker_spec.ts
@@ -17,7 +17,7 @@ import {startWorker} from '../../../src/execution/cluster/worker';
 import {Task, TaskCompletedCallback, TaskProcessingOutcome} from '../../../src/execution/tasks/api';
 import {FileToWrite} from '../../../src/rendering/utils';
 import {MockLogger} from '../../helpers/mock_logger';
-import {mockProperty} from '../../helpers/spy_utils';
+import {mockProperty, spyProperty} from '../../helpers/spy_utils';
 
 
 describe('startWorker()', () => {
@@ -163,6 +163,38 @@ describe('startWorker()', () => {
           .toHaveBeenCalledWith({type: 'error', error: err.stack}, jasmine.any(Function));
     });
 
+    it('should exit on `ENOMEM` errors during task processing', () => {
+      const processExitSpy = jasmine.createSpy('process.exit');
+      const {
+        setMockValue: mockProcessExit,
+        installSpies: installProcessExitSpies,
+        uninstallSpies: uninstallProcessExitSpies,
+      } = spyProperty(process, 'exit');
+
+      try {
+        installProcessExitSpies();
+        mockProcessExit(processExitSpy as unknown as typeof process.exit);
+
+        const mockTask = {
+          entryPoint: {name: 'foo'},
+          formatProperty: 'es2015',
+          processDts: true,
+        } as unknown as Task;
+
+        const noMemError = Object.assign(new Error('ENOMEM: not enough memory'), {code: 'ENOMEM'});
+        compileFnSpy.and.throwError(noMemError);
+
+        startWorker(mockLogger, createCompileFnSpy);
+        cluster.worker.emit('message', {type: 'process-task', task: mockTask});
+
+        expect(mockLogger.logs.warn).toEqual([[`[Worker #42] ${noMemError.stack}`]]);
+        expect(processExitSpy).toHaveBeenCalledWith(1);
+        expect(processSendSpy).not.toHaveBeenCalled();
+      } finally {
+        uninstallProcessExitSpies();
+      }
+    });
+
     it('should throw, when an unknown message type is received', () => {
       startWorker(mockLogger, createCompileFnSpy);
       cluster.worker.emit('message', {type: 'unknown', foo: 'bar'});