iSharkFly-Docs
/
angular-docs-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(ngcc): use the `EntryPointManifest` in `DirectoryWalkerEntryPointFinder` (#35931) 

The `DirectoryWalkerEntryPointFinder` has to traverse the
entire node_modules library everytime it executes in order to
identify the entry-points that need to be processed. This is
very time consuming (several seconds for big projects on
Windows).

This commit changes the `DirectoryWalkerEntryPointFinder` to
use the `EntryPointManifest` to store the paths to entry-points
that were found when doing this initial node_modules traversal
in a file to be reused for subsequent calls.

This dramatically speeds up ngcc processing when it has been run once
already.

PR Close #35931
This commit is contained in:
Pete Bacon Darwin 2020-03-10 10:49:17 +00:00 committed by Andrew Kushnir
parent 560542c2a8
commit ec9f4d5bc6
3 changed files with 106 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
packages/compiler-cli/ngcc/src/entry_point_finder/directory_walker_entry_point_finder.ts
View File

@ -10,6 +10,7 @@ import {DependencyResolver, SortedEntryPointsInfo} from '../dependencies/depende
import {Logger} from '../logging/logger';
import {NgccConfiguration} from '../packages/configuration';
import {EntryPoint, INVALID_ENTRY_POINT, NO_ENTRY_POINT, getEntryPointInfo} from '../packages/entry_point';
import {EntryPointManifest} from '../packages/entry_point_manifest';
import {PathMappings} from '../utils';
import {NGCC_DIRECTORY} from '../writing/new_entry_point_file_writer';
import {EntryPointFinder} from './interface';
@ -23,16 +24,28 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
private basePaths = getBasePaths(this.sourceDirectory, this.pathMappings);
constructor(
private fs: FileSystem, private config: NgccConfiguration, private logger: Logger,
private resolver: DependencyResolver, private sourceDirectory: AbsoluteFsPath,
private pathMappings: PathMappings|undefined) {}
private resolver: DependencyResolver, private entryPointManifest: EntryPointManifest,
private sourceDirectory: AbsoluteFsPath, private pathMappings: PathMappings|undefined) {}
/**
* Search the `sourceDirectory`, and sub-directories, using `pathMappings` as necessary, to find
* all package entry-points.
*/
findEntryPoints(): SortedEntryPointsInfo {
const unsortedEntryPoints = this.basePaths.reduce<EntryPoint[]>(
(entryPoints, basePath) => entryPoints.concat(this.walkDirectoryForEntryPoints(basePath)),
[]);
const unsortedEntryPoints: EntryPoint[] = [];
for (const basePath of this.basePaths) {
let entryPoints = this.entryPointManifest.readEntryPointsUsingManifest(basePath);
if (entryPoints === null) {
this.logger.debug(
`No manifest found for ${basePath} so walking the directories for entry-points.`);
const startTime = Date.now();
entryPoints = this.walkDirectoryForEntryPoints(basePath);
const duration = Math.round((Date.now() - startTime) / 100) / 10;
this.logger.debug(`Walking directories took ${duration}s.`);
this.entryPointManifest.writeEntryPointManifest(basePath, entryPoints);
}
unsortedEntryPoints.push(...entryPoints);
}
return this.resolver.sortEntryPointsByDependency(unsortedEntryPoints);
}

11
packages/compiler-cli/ngcc/src/main.ts
View File

@ -39,6 +39,7 @@ import {hasBeenProcessed} from './packages/build_marker';
import {NgccConfiguration} from './packages/configuration';
import {EntryPoint, EntryPointJsonProperty, EntryPointPackageJson, SUPPORTED_FORMAT_PROPERTIES, getEntryPointFormat} from './packages/entry_point';
import {makeEntryPointBundle} from './packages/entry_point_bundle';
import {EntryPointManifest} from './packages/entry_point_manifest';
import {Transformer} from './packages/transformer';
import {PathMappings} from './utils';
import {cleanOutdatedPackages} from './writing/cleaning/package_cleaner';
@ -153,14 +154,15 @@ export function mainNgcc(
const absBasePath = absoluteFrom(basePath);
const config = new NgccConfiguration(fileSystem, dirname(absBasePath));
const dependencyResolver = getDependencyResolver(fileSystem, logger, config, pathMappings);
const entryPointManifest = new EntryPointManifest(fileSystem, config, logger);
// Bail out early if the work is already done.
const supportedPropertiesToConsider = ensureSupportedProperties(propertiesToConsider);
const absoluteTargetEntryPointPath =
targetEntryPointPath !== undefined ? resolve(basePath, targetEntryPointPath) : null;
const finder = getEntryPointFinder(
fileSystem, logger, dependencyResolver, config, absBasePath, absoluteTargetEntryPointPath,
pathMappings);
fileSystem, logger, dependencyResolver, config, entryPointManifest, absBasePath,
absoluteTargetEntryPointPath, pathMappings);
if (finder instanceof TargetedEntryPointFinder &&
!finder.targetNeedsProcessingOrCleaning(supportedPropertiesToConsider, compileAllFormats)) {
logger.debug('The target entry-point has already been processed');
@ -376,14 +378,15 @@ function getDependencyResolver(
function getEntryPointFinder(
fs: FileSystem, logger: Logger, resolver: DependencyResolver, config: NgccConfiguration,
basePath: AbsoluteFsPath, absoluteTargetEntryPointPath: AbsoluteFsPath | null,
entryPointManifest: EntryPointManifest, basePath: AbsoluteFsPath,
absoluteTargetEntryPointPath: AbsoluteFsPath | null,
pathMappings: PathMappings | undefined): EntryPointFinder {
if (absoluteTargetEntryPointPath !== null) {
return new TargetedEntryPointFinder(
fs, config, logger, resolver, basePath, absoluteTargetEntryPointPath, pathMappings);
} else {
return new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, basePath, pathMappings);
fs, config, logger, resolver, entryPointManifest, basePath, pathMappings);
}
}

98
packages/compiler-cli/ngcc/test/entry_point_finder/directory_walker_entry_point_finder_spec.ts
View File

@ -15,6 +15,7 @@ import {ModuleResolver} from '../../src/dependencies/module_resolver';
import {DirectoryWalkerEntryPointFinder} from '../../src/entry_point_finder/directory_walker_entry_point_finder';
import {NgccConfiguration} from '../../src/packages/configuration';
import {EntryPoint} from '../../src/packages/entry_point';
import {EntryPointManifest, EntryPointManifestFile} from '../../src/packages/entry_point_manifest';
import {PathMappings} from '../../src/utils';
import {MockLogger} from '../helpers/mock_logger';
@ -24,6 +25,7 @@ runInEachFileSystem(() => {
let resolver: DependencyResolver;
let logger: MockLogger;
let config: NgccConfiguration;
let manifest: EntryPointManifest;
let _Abs: typeof absoluteFrom;
beforeEach(() => {
@ -34,6 +36,7 @@ runInEachFileSystem(() => {
const dtsHost = new DtsDependencyHost(fs);
config = new NgccConfiguration(fs, _Abs('/'));
resolver = new DependencyResolver(fs, logger, config, {esm2015: srcHost}, dtsHost);
manifest = new EntryPointManifest(fs, config, logger);
});
describe('findEntryPoints()', () => {
@ -46,8 +49,8 @@ runInEachFileSystem(() => {
fs.resolve(basePath, 'common/http'), 'testing', ['common/http', 'common/testing']),
...createPackage(fs.resolve(basePath, 'common'), 'testing', ['common']),
]);
const finder =
new DirectoryWalkerEntryPointFinder(fs, config, logger, resolver, basePath, undefined);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, manifest, basePath, undefined);
const {entryPoints} = finder.findEntryPoints();
expect(dumpEntryPointPaths(basePath, entryPoints)).toEqual([
['common', 'common'],
@ -67,8 +70,8 @@ runInEachFileSystem(() => {
['@angular/common/http', '@angular/common/testing']),
...createPackage(fs.resolve(basePath, '@angular/common'), 'testing', ['@angular/common']),
]);
const finder =
new DirectoryWalkerEntryPointFinder(fs, config, logger, resolver, basePath, undefined);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, manifest, basePath, undefined);
const {entryPoints} = finder.findEntryPoints();
expect(dumpEntryPointPaths(basePath, entryPoints)).toEqual([
['@angular/common', '@angular/common'],
@ -81,7 +84,7 @@ runInEachFileSystem(() => {
it('should return an empty array if there are no packages', () => {
fs.ensureDir(_Abs('/no_packages/node_modules/should_not_be_found'));
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, _Abs('/no_packages/node_modules'), undefined);
fs, config, logger, resolver, manifest, _Abs('/no_packages/node_modules'), undefined);
const {entryPoints} = finder.findEntryPoints();
expect(entryPoints).toEqual([]);
});
@ -94,17 +97,75 @@ runInEachFileSystem(() => {
},
]);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, _Abs('/no_valid_entry_points/node_modules'), undefined);
fs, config, logger, resolver, manifest, _Abs('/no_valid_entry_points/node_modules'),
undefined);
const {entryPoints} = finder.findEntryPoints();
expect(entryPoints).toEqual([]);
});
it('should write an entry-point manifest file if none was found', () => {
const basePath = _Abs('/sub_entry_points/node_modules');
loadTestFiles([
...createPackage(basePath, 'common'),
...createPackage(fs.resolve(basePath, 'common'), 'http', ['common']),
...createPackage(
fs.resolve(basePath, 'common/http'), 'testing', ['common/http', 'common/testing']),
...createPackage(fs.resolve(basePath, 'common'), 'testing', ['common']),
{name: _Abs('/sub_entry_points/yarn.lock'), contents: 'MOCM LOCK FILE'},
]);
spyOn(manifest, 'readEntryPointsUsingManifest').and.callThrough();
spyOn(manifest, 'writeEntryPointManifest').and.callThrough();
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, manifest, basePath, undefined);
finder.findEntryPoints();
expect(manifest.readEntryPointsUsingManifest).toHaveBeenCalled();
expect(manifest.writeEntryPointManifest).toHaveBeenCalled();
expect(fs.exists(_Abs('/sub_entry_points/node_modules/__ngcc_entry_points__.json')))
.toBe(true);
});
it('should read from the entry-point manifest file if found', () => {
const basePath = _Abs('/sub_entry_points/node_modules');
loadTestFiles([
...createPackage(basePath, 'common'),
...createPackage(fs.resolve(basePath, 'common'), 'http', ['common']),
...createPackage(
fs.resolve(basePath, 'common/http'), 'testing', ['common/http', 'common/testing']),
...createPackage(fs.resolve(basePath, 'common'), 'testing', ['common']),
{name: _Abs('/sub_entry_points/yarn.lock'), contents: 'MOCM LOCK FILE'},
]);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, manifest, basePath, undefined);
// Prime the manifest by calling findEntryPoints() once.
finder.findEntryPoints();
// Modify the manifest to prove that we use it to find the entry-points
const manifestPath = _Abs('/sub_entry_points/node_modules/__ngcc_entry_points__.json');
const manifestFile: EntryPointManifestFile = JSON.parse(fs.readFile(manifestPath));
manifestFile.entryPointPaths.pop();
fs.writeFile(manifestPath, JSON.stringify(manifestFile));
// Now see if the manifest is read on a second call.
spyOn(manifest, 'readEntryPointsUsingManifest').and.callThrough();
spyOn(manifest, 'writeEntryPointManifest').and.callThrough();
const {entryPoints} = finder.findEntryPoints();
expect(manifest.readEntryPointsUsingManifest).toHaveBeenCalled();
expect(manifest.writeEntryPointManifest).not.toHaveBeenCalled();
expect(dumpEntryPointPaths(basePath, entryPoints)).toEqual([
['common', 'common'],
['common', 'common/http'],
['common', 'common/http/testing'],
]);
});
it('should ignore folders starting with .', () => {
loadTestFiles([
...createPackage(_Abs('/dotted_folders/node_modules/'), '.common'),
]);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, _Abs('/dotted_folders/node_modules'), undefined);
fs, config, logger, resolver, manifest, _Abs('/dotted_folders/node_modules'),
undefined);
const {entryPoints} = finder.findEntryPoints();
expect(entryPoints).toEqual([]);
});
@ -115,7 +176,8 @@ runInEachFileSystem(() => {
_Abs('/external/node_modules/common'), _Abs('/symlinked_folders/node_modules/common'));
loadTestFiles(createPackage(_Abs('/external/node_modules'), 'common'));
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, _Abs('/symlinked_folders/node_modules'), undefined);
fs, config, logger, resolver, manifest, _Abs('/symlinked_folders/node_modules'),
undefined);
const {entryPoints} = finder.findEntryPoints();
expect(entryPoints).toEqual([]);
});
@ -126,7 +188,8 @@ runInEachFileSystem(() => {
...createPackage(_Abs('/nested_node_modules/node_modules/outer/node_modules'), 'inner'),
]);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, _Abs('/nested_node_modules/node_modules'), undefined);
fs, config, logger, resolver, manifest, _Abs('/nested_node_modules/node_modules'),
undefined);
const {entryPoints} = finder.findEntryPoints();
// Note that the `inner` entry-point is not part of the `outer` package
expect(dumpEntryPointPaths(_Abs('/nested_node_modules/node_modules'), entryPoints))
@ -136,7 +199,7 @@ runInEachFileSystem(() => {
]);
});
it('should handle try to process nested node_modules of non Angular packages', () => {
it('should not try to process nested node_modules of non Angular packages', () => {
const basePath = _Abs('/nested_node_modules/node_modules');
loadTestFiles([
...createPackage(basePath, 'outer', ['inner'], false),
@ -144,7 +207,8 @@ runInEachFileSystem(() => {
]);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, _Abs('/nested_node_modules/node_modules'), undefined);
fs, config, logger, resolver, manifest, _Abs('/nested_node_modules/node_modules'),
undefined);
const spy = spyOn(finder, 'walkDirectoryForEntryPoints').and.callThrough();
const {entryPoints} = finder.findEntryPoints();
expect(spy.calls.allArgs()).toEqual([
@ -165,8 +229,8 @@ runInEachFileSystem(() => {
},
]);
const finder =
new DirectoryWalkerEntryPointFinder(fs, config, logger, resolver, basePath, undefined);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, manifest, basePath, undefined);
const spy = spyOn(finder, 'walkDirectoryForEntryPoints').and.callThrough();
const {entryPoints} = finder.findEntryPoints();
expect(spy.calls.allArgs()).toEqual([
@ -189,8 +253,8 @@ runInEachFileSystem(() => {
},
]);
const finder =
new DirectoryWalkerEntryPointFinder(fs, config, logger, resolver, basePath, undefined);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, manifest, basePath, undefined);
const spy = spyOn(finder, 'walkDirectoryForEntryPoints').and.callThrough();
const {entryPoints} = finder.findEntryPoints();
expect(spy.calls.allArgs()).toEqual([
@ -223,7 +287,7 @@ runInEachFileSystem(() => {
const dtsHost = new DtsDependencyHost(fs, pathMappings);
resolver = new DependencyResolver(fs, logger, config, {esm2015: srcHost}, dtsHost);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, basePath, pathMappings);
fs, config, logger, resolver, manifest, basePath, pathMappings);
const {entryPoints} = finder.findEntryPoints();
expect(dumpEntryPointPaths(basePath, entryPoints)).toEqual([
['pkg1', 'pkg1'],
@ -251,7 +315,7 @@ runInEachFileSystem(() => {
const dtsHost = new DtsDependencyHost(fs, pathMappings);
resolver = new DependencyResolver(fs, logger, config, {esm2015: srcHost}, dtsHost);
const finder = new DirectoryWalkerEntryPointFinder(
fs, config, logger, resolver, basePath, pathMappings);
fs, config, logger, resolver, manifest, basePath, pathMappings);
const {entryPoints} = finder.findEntryPoints();
expect(dumpEntryPointPaths(basePath, entryPoints)).toEqual([
['test', 'test'],