From e790c8547e1f5f64ac84976ed5ba07e9888cdc62 Mon Sep 17 00:00:00 2001 From: JoostK Date: Sun, 20 Sep 2020 01:06:19 +0200 Subject: [PATCH] test(compiler-cli): load test files into memory only once (#38909) Prior to this change, each invocation of `loadStandardTestFiles` would load the necessary files from disk. This function is typically called at the top-level of a test module in order to share the result across tests. The `//packages/compiler-cli/test/ngtsc` target has 8 modules where this call occurs, each loading their own copy of `node_modules/typescript` which is ~60MB in size, so the memory overhead used to be significant. This commit loads the individual packages into a standalone `Folder` and mounts this folder into the filesystem of standard test files, such that all file contents are no longer duplicated in memory. PR Close #38909 --- .../testing/src/mock_file_system.ts | 45 ++++++++++++++----- .../test/helpers/src/mock_file_loading.ts | 45 +++++++++++++++---- 2 files changed, 72 insertions(+), 18 deletions(-) diff --git a/packages/compiler-cli/src/ngtsc/file_system/testing/src/mock_file_system.ts b/packages/compiler-cli/src/ngtsc/file_system/testing/src/mock_file_system.ts index 9122f531eb..220ac4e0d5 100644 --- a/packages/compiler-cli/src/ngtsc/file_system/testing/src/mock_file_system.ts +++ b/packages/compiler-cli/src/ngtsc/file_system/testing/src/mock_file_system.ts @@ -127,12 +127,17 @@ export abstract class MockFileSystem implements FileSystem { delete folder[name]; } - ensureDir(path: AbsoluteFsPath): void { + ensureDir(path: AbsoluteFsPath): Folder { const segments = this.splitPath(path).map(segment => this.getCanonicalPath(segment)); - let current: Folder = this._fileTree; // Convert the root folder to a canonical empty string `''` (on Windows it would be `'C:'`). segments[0] = ''; + if (segments.length > 1 && segments[segments.length - 1] === '') { + // Remove a trailing slash (unless the path was only `/`) + segments.pop(); + } + + let current: Folder = this._fileTree; for (const segment of segments) { if (isFile(current[segment])) { throw new Error(`Folder already exists as a file.`); @@ -142,6 +147,7 @@ export abstract class MockFileSystem implements FileSystem { } current = current[segment] as Folder; } + return current; } removeDeep(path: AbsoluteFsPath): void { @@ -221,26 +227,45 @@ export abstract class MockFileSystem implements FileSystem { protected abstract splitPath(path: T): string[]; dump(): Folder { - return this.cloneFolder(this._fileTree); + const {entity} = this.findFromPath(this.resolve('/')); + if (entity === null || !isFolder(entity)) { + return {}; + } + + return this.cloneFolder(entity); } + init(folder: Folder): void { - this._fileTree = this.cloneFolder(folder); + this.mount(this.resolve('/'), folder); + } + + mount(path: AbsoluteFsPath, folder: Folder): void { + if (this.exists(path)) { + throw new Error(`Unable to mount in '${path}' as it already exists.`); + } + const mountFolder = this.ensureDir(path); + + this.copyInto(folder, mountFolder); } private cloneFolder(folder: Folder): Folder { const clone: Folder = {}; - for (const path in folder) { - const item = folder[path]; + this.copyInto(folder, clone); + return clone; + } + + private copyInto(from: Folder, to: Folder): void { + for (const path in from) { + const item = from[path]; const canonicalPath = this.getCanonicalPath(path); if (isSymLink(item)) { - clone[canonicalPath] = new SymLink(this.getCanonicalPath(item.path)); + to[canonicalPath] = new SymLink(this.getCanonicalPath(item.path)); } else if (isFolder(item)) { - clone[canonicalPath] = this.cloneFolder(item); + to[canonicalPath] = this.cloneFolder(item); } else { - clone[canonicalPath] = folder[path]; + to[canonicalPath] = from[path]; } } - return clone; } diff --git a/packages/compiler-cli/test/helpers/src/mock_file_loading.ts b/packages/compiler-cli/test/helpers/src/mock_file_loading.ts index e5e8cb79b9..749e57c93d 100644 --- a/packages/compiler-cli/test/helpers/src/mock_file_loading.ts +++ b/packages/compiler-cli/test/helpers/src/mock_file_loading.ts @@ -22,28 +22,43 @@ export function loadTestFiles(files: TestFile[]) { }); } +/** + * A folder that is lazily loaded upon first access and then cached. + */ +class CachedFolder { + private folder: Folder|null = null; + + constructor(private loader: () => Folder) {} + + get(): Folder { + if (this.folder === null) { + this.folder = this.loader(); + } + return this.folder; + } +} + +const typescriptFolder = new CachedFolder(() => loadFolder(resolveNpmTreeArtifact('typescript'))); +const angularFolder = new CachedFolder(loadAngularFolder); +const rxjsFolder = new CachedFolder(() => loadFolder(resolveNpmTreeArtifact('rxjs'))); + export function loadStandardTestFiles( {fakeCore = true, rxjs = false}: {fakeCore?: boolean, rxjs?: boolean} = {}): Folder { const tmpFs = new MockFileSystemPosix(true); const basePath = '/' as AbsoluteFsPath; - loadTestDirectory( - tmpFs, resolveNpmTreeArtifact('typescript'), - tmpFs.resolve(basePath, 'node_modules/typescript')); + tmpFs.mount(tmpFs.resolve('/node_modules/typescript'), typescriptFolder.get()); loadTsLib(tmpFs, basePath); if (fakeCore) { loadFakeCore(tmpFs, basePath); } else { - getAngularPackagesFromRunfiles().forEach(({name, pkgPath}) => { - loadTestDirectory(tmpFs, pkgPath, tmpFs.resolve(basePath, 'node_modules/@angular', name)); - }); + tmpFs.mount(tmpFs.resolve('/node_modules/@angular'), angularFolder.get()); } if (rxjs) { - loadTestDirectory( - tmpFs, resolveNpmTreeArtifact('rxjs'), tmpFs.resolve(basePath, 'node_modules/rxjs')); + tmpFs.mount(tmpFs.resolve('/node_modules/rxjs'), rxjsFolder.get()); } return tmpFs.dump(); @@ -60,6 +75,20 @@ export function loadFakeCore(fs: FileSystem, basePath: string = '/') { fs.resolve(basePath, 'node_modules/@angular/core')); } +function loadFolder(path: string): Folder { + const tmpFs = new MockFileSystemPosix(true); + loadTestDirectory(tmpFs, tmpFs.resolve(path), tmpFs.resolve('/')); + return tmpFs.dump(); +} + +function loadAngularFolder(): Folder { + const tmpFs = new MockFileSystemPosix(true); + getAngularPackagesFromRunfiles().forEach(({name, pkgPath}) => { + loadTestDirectory(tmpFs, pkgPath, tmpFs.resolve(name)); + }); + return tmpFs.dump(); +} + /** * Load real files from the real file-system into a mock file-system. * @param fs the file-system where the directory is to be loaded.