236 lines
8.5 KiB
TypeScript
Raw Normal View History

/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {absoluteFrom, AbsoluteFsPath, FileSystem, getFileSystem} from '../../src/ngtsc/file_system';
import {ConsoleLogger, Logger, LogLevel} from '../../src/ngtsc/logging';
import {ParsedConfiguration, readConfiguration} from '../../src/perform_compile';
import {SUPPORTED_FORMAT_PROPERTIES} from './packages/entry_point';
import {getPathMappingsFromTsConfig, PathMappings} from './path_mappings';
fix(ngcc): support recovering when a worker process crashes (#36626) Previously, when running in parallel mode and a worker process crashed while processing a task, it was not possible for ngcc to continue without risking ending up with a corrupted entry-point and therefore it exited with an error. This, for example, could happen when a worker process received a `SIGKILL` signal, which was frequently observed in CI environments. This was probably the result of Docker killing processes due to increased memory pressure. One factor that amplifies the problem under Docker (which is often used in CI) is that it is not possible to distinguish between the available CPU cores on the host machine and the ones made available to Docker containers, thus resulting in ngcc spawning too many worker processes. This commit addresses these issues in the following ways: 1. We take advantage of the fact that files are written to disk only after an entry-point has been fully analyzed/compiled. The master process can now determine whether a worker process has not yet started writing files to disk (even if it was in the middle of processing a task) and just put the task back into the tasks queue if the worker process crashes. 2. The master process keeps track of the transformed files that a worker process will attempt to write to disk. If the worker process crashes while writing files, the master process can revert any changes and put the task back into the tasks queue (without risking corruption). 3. When a worker process crashes while processing a task (which can be a result of increased memory pressure or too many worker processes), the master process will not try to re-spawn it. This way the number or worker processes is gradually adjusted to a level that can be accomodated by the system's resources. Examples of ngcc being able to recover after a worker process crashed: - While idling: https://circleci.com/gh/angular/angular/682197 - While compiling: https://circleci.com/gh/angular/angular/682209 - While writing files: https://circleci.com/gh/angular/angular/682267 Jira issue: [FW-2008](https://angular-team.atlassian.net/browse/FW-2008) Fixes #36278 PR Close #36626
2020-04-29 21:28:22 +03:00
import {FileWriter} from './writing/file_writer';
import {InPlaceFileWriter} from './writing/in_place_file_writer';
import {NewEntryPointFileWriter} from './writing/new_entry_point_file_writer';
import {PackageJsonUpdater} from './writing/package_json_updater';
/**
* The options to configure the ngcc compiler for synchronous execution.
*/
export interface SyncNgccOptions {
/** The absolute path to the `node_modules` folder that contains the packages to process. */
basePath: string;
/**
* The path to the primary package to be processed. If not absolute then it must be relative to
* `basePath`.
*
* All its dependencies will need to be processed too.
*
* If this property is provided then `errorOnFailedEntryPoint` is forced to true.
*/
targetEntryPointPath?: string;
/**
* Which entry-point properties in the package.json to consider when processing an entry-point.
* Each property should hold a path to the particular bundle format for the entry-point.
* Defaults to all the properties in the package.json.
*/
propertiesToConsider?: string[];
/**
* Whether to process all formats specified by (`propertiesToConsider`) or to stop processing
* this entry-point at the first matching format. Defaults to `true`.
*/
compileAllFormats?: boolean;
/**
* Whether to create new entry-points bundles rather than overwriting the original files.
*/
createNewEntryPointFormats?: boolean;
/**
* Provide a logger that will be called with log messages.
*/
logger?: Logger;
/**
* Paths mapping configuration (`paths` and `baseUrl`), as found in `ts.CompilerOptions`.
* These are used to resolve paths to locally built Angular libraries.
*
* Note that `pathMappings` specified here take precedence over any `pathMappings` loaded from a
* TS config file.
*/
pathMappings?: PathMappings;
/**
* Provide a file-system service that will be used by ngcc for all file interactions.
*/
fileSystem?: FileSystem;
/**
* Whether the compilation should run and return asynchronously. Allowing asynchronous execution
* may speed up the compilation by utilizing multiple CPU cores (if available).
*
* Default: `false` (i.e. run synchronously)
*/
async?: false;
/**
* Set to true in order to terminate immediately with an error code if an entry-point fails to be
* processed.
*
* If `targetEntryPointPath` is provided then this property is always true and cannot be
* changed. Otherwise the default is false.
*
* When set to false, ngcc will continue to process entry-points after a failure. In which case it
* will log an error and resume processing other entry-points.
*/
errorOnFailedEntryPoint?: boolean;
/**
* Render `$localize` messages with legacy format ids.
*
* The default value is `true`. Only set this to `false` if you do not want legacy message ids to
* be rendered. For example, if you are not using legacy message ids in your translation files
* AND are not doing compile-time inlining of translations, in which case the extra message ids
* would add unwanted size to the final source bundle.
*
* It is safe to leave this set to true if you are doing compile-time inlining because the extra
* legacy message ids will all be stripped during translation.
*/
enableI18nLegacyMessageIdFormat?: boolean;
/**
* Whether to invalidate any entry-point manifest file that is on disk. Instead, walk the
* directory tree looking for entry-points, and then write a new entry-point manifest, if
* possible.
*
* Default: `false` (i.e. the manifest will be used if available)
*/
invalidateEntryPointManifest?: boolean;
/**
* An absolute path to a TS config file (e.g. `tsconfig.json`) or a directory containing one, that
* will be used to configure module resolution with things like path mappings, if not specified
* explicitly via the `pathMappings` property to `mainNgcc`.
*
* If `undefined`, ngcc will attempt to load a `tsconfig.json` file from the directory above the
* `basePath`.
*
* If `null`, ngcc will not attempt to load any TS config file at all.
*/
tsConfigPath?: string|null;
/**
* Use the program defined in the loaded tsconfig.json (if available - see
* `tsConfigPath` option) to identify the entry-points that should be processed.
* If this is set to `true` then only the entry-points reachable from the given
* program (and their dependencies) will be processed.
*/
findEntryPointsFromTsConfigProgram?: boolean;
}
/**
* The options to configure the ngcc compiler for asynchronous execution.
*/
export type AsyncNgccOptions = Omit<SyncNgccOptions, 'async'>&{async: true};
/**
* The options to configure the ngcc compiler.
*/
export type NgccOptions = AsyncNgccOptions|SyncNgccOptions;
export type OptionalNgccOptionKeys =
'targetEntryPointPath'|'tsConfigPath'|'pathMappings'|'findEntryPointsFromTsConfigProgram';
export type RequiredNgccOptions = Required<Omit<NgccOptions, OptionalNgccOptionKeys>>;
export type OptionalNgccOptions = Pick<NgccOptions, OptionalNgccOptionKeys>;
export type SharedSetup = {
fileSystem: FileSystem; absBasePath: AbsoluteFsPath; projectPath: AbsoluteFsPath;
tsConfig: ParsedConfiguration | null;
fix(ngcc): support recovering when a worker process crashes (#36626) Previously, when running in parallel mode and a worker process crashed while processing a task, it was not possible for ngcc to continue without risking ending up with a corrupted entry-point and therefore it exited with an error. This, for example, could happen when a worker process received a `SIGKILL` signal, which was frequently observed in CI environments. This was probably the result of Docker killing processes due to increased memory pressure. One factor that amplifies the problem under Docker (which is often used in CI) is that it is not possible to distinguish between the available CPU cores on the host machine and the ones made available to Docker containers, thus resulting in ngcc spawning too many worker processes. This commit addresses these issues in the following ways: 1. We take advantage of the fact that files are written to disk only after an entry-point has been fully analyzed/compiled. The master process can now determine whether a worker process has not yet started writing files to disk (even if it was in the middle of processing a task) and just put the task back into the tasks queue if the worker process crashes. 2. The master process keeps track of the transformed files that a worker process will attempt to write to disk. If the worker process crashes while writing files, the master process can revert any changes and put the task back into the tasks queue (without risking corruption). 3. When a worker process crashes while processing a task (which can be a result of increased memory pressure or too many worker processes), the master process will not try to re-spawn it. This way the number or worker processes is gradually adjusted to a level that can be accomodated by the system's resources. Examples of ngcc being able to recover after a worker process crashed: - While idling: https://circleci.com/gh/angular/angular/682197 - While compiling: https://circleci.com/gh/angular/angular/682209 - While writing files: https://circleci.com/gh/angular/angular/682267 Jira issue: [FW-2008](https://angular-team.atlassian.net/browse/FW-2008) Fixes #36278 PR Close #36626
2020-04-29 21:28:22 +03:00
getFileWriter(pkgJsonUpdater: PackageJsonUpdater): FileWriter;
};
/**
* Instantiate common utilities that are always used and fix up options with defaults, as necessary.
*
* NOTE: Avoid eagerly instantiating anything that might not be used when running sync/async.
*/
export function getSharedSetup(options: NgccOptions): SharedSetup&RequiredNgccOptions&
OptionalNgccOptions {
const fileSystem = getFileSystem();
const absBasePath = absoluteFrom(options.basePath);
const projectPath = fileSystem.dirname(absBasePath);
const tsConfig =
options.tsConfigPath !== null ? getTsConfig(options.tsConfigPath || projectPath) : null;
let {
basePath,
targetEntryPointPath,
propertiesToConsider = SUPPORTED_FORMAT_PROPERTIES,
compileAllFormats = true,
createNewEntryPointFormats = false,
logger = new ConsoleLogger(LogLevel.info),
pathMappings = getPathMappingsFromTsConfig(tsConfig, projectPath),
async = false,
errorOnFailedEntryPoint = false,
enableI18nLegacyMessageIdFormat = true,
invalidateEntryPointManifest = false,
tsConfigPath,
} = options;
if (!!targetEntryPointPath) {
// targetEntryPointPath forces us to error if an entry-point fails.
errorOnFailedEntryPoint = true;
}
return {
basePath,
targetEntryPointPath,
propertiesToConsider,
compileAllFormats,
createNewEntryPointFormats,
logger,
pathMappings,
async,
errorOnFailedEntryPoint,
enableI18nLegacyMessageIdFormat,
invalidateEntryPointManifest,
tsConfigPath,
fileSystem,
absBasePath,
projectPath,
tsConfig,
fix(ngcc): support recovering when a worker process crashes (#36626) Previously, when running in parallel mode and a worker process crashed while processing a task, it was not possible for ngcc to continue without risking ending up with a corrupted entry-point and therefore it exited with an error. This, for example, could happen when a worker process received a `SIGKILL` signal, which was frequently observed in CI environments. This was probably the result of Docker killing processes due to increased memory pressure. One factor that amplifies the problem under Docker (which is often used in CI) is that it is not possible to distinguish between the available CPU cores on the host machine and the ones made available to Docker containers, thus resulting in ngcc spawning too many worker processes. This commit addresses these issues in the following ways: 1. We take advantage of the fact that files are written to disk only after an entry-point has been fully analyzed/compiled. The master process can now determine whether a worker process has not yet started writing files to disk (even if it was in the middle of processing a task) and just put the task back into the tasks queue if the worker process crashes. 2. The master process keeps track of the transformed files that a worker process will attempt to write to disk. If the worker process crashes while writing files, the master process can revert any changes and put the task back into the tasks queue (without risking corruption). 3. When a worker process crashes while processing a task (which can be a result of increased memory pressure or too many worker processes), the master process will not try to re-spawn it. This way the number or worker processes is gradually adjusted to a level that can be accomodated by the system's resources. Examples of ngcc being able to recover after a worker process crashed: - While idling: https://circleci.com/gh/angular/angular/682197 - While compiling: https://circleci.com/gh/angular/angular/682209 - While writing files: https://circleci.com/gh/angular/angular/682267 Jira issue: [FW-2008](https://angular-team.atlassian.net/browse/FW-2008) Fixes #36278 PR Close #36626
2020-04-29 21:28:22 +03:00
getFileWriter: (pkgJsonUpdater: PackageJsonUpdater) => createNewEntryPointFormats ?
new NewEntryPointFileWriter(fileSystem, logger, errorOnFailedEntryPoint, pkgJsonUpdater) :
new InPlaceFileWriter(fileSystem, logger, errorOnFailedEntryPoint),
};
}
let tsConfigCache: ParsedConfiguration|null = null;
let tsConfigPathCache: string|null = null;
/**
* Get the parsed configuration object for the given `tsConfigPath`.
*
* This function will cache the previous parsed configuration object to avoid unnecessary processing
* of the tsconfig.json in the case that it is requested repeatedly.
*
* This makes the assumption, which is true as of writing, that the contents of tsconfig.json and
* its dependencies will not change during the life of the process running ngcc.
*/
function getTsConfig(tsConfigPath: string): ParsedConfiguration|null {
if (tsConfigPath !== tsConfigPathCache) {
tsConfigPathCache = tsConfigPath;
tsConfigCache = readConfiguration(tsConfigPath);
}
return tsConfigCache;
}
export function clearTsConfigCache() {
tsConfigPathCache = null;
tsConfigCache = null;
}