refactor(ngcc): add entry-point manifest functionality (#35931)

The new `EntryPointManifest` class can read and write a
manifest file that contains all the paths to the entry-points
that have been found in a node_modules folder.
This can be used to speed up finding entry-points in
subsequent runs.

The manifest file stores the ngcc version and hashes of
the package lock-file and project config, since if these
change the manifest will need to be recomputed.

PR Close #35931

packages/compiler-cli/ngcc/src/packages/entry_point_manifest.ts

@@ -0,0 +1,140 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {createHash} from 'crypto';
+
+import {AbsoluteFsPath, FileSystem} from '../../../src/ngtsc/file_system';
+import {Logger} from '../logging/logger';
+
+import {NGCC_VERSION} from './build_marker';
+import {NgccConfiguration} from './configuration';
+import {EntryPoint, INVALID_ENTRY_POINT, NO_ENTRY_POINT, getEntryPointInfo} from './entry_point';
+
+/**
+ * Manages reading and writing a manifest file that contains a list of all the entry-points that
+ * were found below a given basePath.
+ *
+ * This is a super-set of the entry-points that are actually processed for a given run of ngcc,
+ * since some may already be processed, or excluded if they do not have the required format.
+ */
+export class EntryPointManifest {
+  constructor(private fs: FileSystem, private config: NgccConfiguration, private logger: Logger) {}
+
+  /**
+   * Try to get the entry-point info from a manifest file for the given `basePath` if it exists and
+   * is not out of date.
+   *
+   * Reasons for the manifest to be out of date are:
+   *
+   * * the file does not exist
+   * * the ngcc version has changed
+   * * the package lock-file (i.e. yarn.lock or package-lock.json) has changed
+   * * the project configuration has changed
+   * * one or more entry-points in the manifest are not valid
+   *
+   * @param basePath The path that would contain the entry-points and the manifest file.
+   * @returns an array of entry-point information for all entry-points found below the given
+   * `basePath` or `null` if the manifest was out of date.
+   */
+  readEntryPointsUsingManifest(basePath: AbsoluteFsPath): EntryPoint[]|null {
+    try {
+      if (this.fs.basename(basePath) !== 'node_modules') {
+        return null;
+      }
+
+      const manifestPath = this.getEntryPointManifestPath(basePath);
+      if (!this.fs.exists(manifestPath)) {
+        return null;
+      }
+
+      const computedLockFileHash = this.computeLockFileHash(basePath);
+      if (computedLockFileHash === null) {
+        return null;
+      }
+
+      const {ngccVersion, configFileHash, lockFileHash, entryPointPaths} =
+          JSON.parse(this.fs.readFile(manifestPath)) as EntryPointManifestFile;
+      if (ngccVersion !== NGCC_VERSION || configFileHash !== this.config.hash ||
+          lockFileHash !== computedLockFileHash) {
+        return null;
+      }
+
+      this.logger.debug(
+          `Entry-point manifest found for ${basePath} so loading entry-point information directly.`);
+      const startTime = Date.now();
+
+      const entryPoints: EntryPoint[] = [];
+      for (const [packagePath, entryPointPath] of entryPointPaths) {
+        const result =
+            getEntryPointInfo(this.fs, this.config, this.logger, packagePath, entryPointPath);
+        if (result === NO_ENTRY_POINT || result === INVALID_ENTRY_POINT) {
+          throw new Error(
+              `The entry-point manifest at ${manifestPath} contained an invalid pair of package paths: [${packagePath}, ${entryPointPath}]`);
+        } else {
+          entryPoints.push(result);
+        }
+      }
+      const duration = Math.round((Date.now() - startTime) / 100) / 10;
+      this.logger.debug(`Reading entry-points using the manifest entries took ${duration}s.`);
+      return entryPoints;
+    } catch (e) {
+      this.logger.warn(
+          `Unable to read the entry-point manifest for ${basePath}:\n`, e.stack || e.toString());
+      return null;
+    }
+  }
+
+  /**
+   * Write a manifest file at the given `basePath`.
+   *
+   * The manifest includes the current ngcc version and hashes of the package lock-file and current
+   * project config. These will be used to check whether the manifest file is out of date. See
+   * `readEntryPointsUsingManifest()`.
+   *
+   * @param basePath The path where the manifest file is to be written.
+   * @param entryPoints A collection of entry-points to record in the manifest.
+   */
+  writeEntryPointManifest(basePath: AbsoluteFsPath, entryPoints: EntryPoint[]): void {
+    const lockFileHash = this.computeLockFileHash(basePath);
+    if (lockFileHash === null) {
+      return;
+    }
+    const manifest: EntryPointManifestFile = {
+      ngccVersion: NGCC_VERSION,
+      configFileHash: this.config.hash,
+      lockFileHash: lockFileHash,
+      entryPointPaths: entryPoints.map(entryPoint => [entryPoint.package, entryPoint.path]),
+    };
+    this.fs.writeFile(this.getEntryPointManifestPath(basePath), JSON.stringify(manifest));
+  }
+
+  private getEntryPointManifestPath(basePath: AbsoluteFsPath) {
+    return this.fs.resolve(basePath, '__ngcc_entry_points__.json');
+  }
+
+  private computeLockFileHash(basePath: AbsoluteFsPath): string|null {
+    const directory = this.fs.dirname(basePath);
+    for (const lockFileName of ['yarn.lock', 'package-lock.json']) {
+      const lockFilePath = this.fs.resolve(directory, lockFileName);
+      if (this.fs.exists(lockFilePath)) {
+        const lockFileContents = this.fs.readFile(lockFilePath);
+        return createHash('md5').update(lockFileContents).digest('hex');
+      }
+    }
+    return null;
+  }
+}
+
+/**
+ * The JSON format of the manifest file that is written to disk.
+ */
+export interface EntryPointManifestFile {
+  ngccVersion: string;
+  configFileHash: string;
+  lockFileHash: string;
+  entryPointPaths: Array<[AbsoluteFsPath, AbsoluteFsPath]>;
+}

packages/compiler-cli/ngcc/test/packages/entry_point_manifest_spec.ts

@@ -0,0 +1,242 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {createHash} from 'crypto';
+
+import {FileSystem, absoluteFrom, getFileSystem} from '../../../src/ngtsc/file_system';
+import {runInEachFileSystem} from '../../../src/ngtsc/file_system/testing';
+import {loadTestFiles} from '../../../test/helpers';
+import {NGCC_VERSION} from '../../src/packages/build_marker';
+import {NgccConfiguration} from '../../src/packages/configuration';
+import {EntryPoint} from '../../src/packages/entry_point';
+import {EntryPointManifest, EntryPointManifestFile} from '../../src/packages/entry_point_manifest';
+import {MockLogger} from '../helpers/mock_logger';
+
+import {createPackageJson} from './entry_point_spec';
+
+runInEachFileSystem(() => {
+  describe('EntryPointManifest', () => {
+    let fs: FileSystem;
+    let _Abs: typeof absoluteFrom;
+    let config: NgccConfiguration;
+    let logger: MockLogger;
+    let manifest: EntryPointManifest;
+
+    beforeEach(() => {
+      _Abs = absoluteFrom;
+      fs = getFileSystem();
+      fs.ensureDir(_Abs('/project/node_modules'));
+      config = new NgccConfiguration(fs, _Abs('/project'));
+      logger = new MockLogger();
+      manifest = new EntryPointManifest(fs, config, logger);
+    });
+
+    describe('readEntryPointsUsingManifest()', () => {
+      let manifestFile: EntryPointManifestFile;
+      beforeEach(() => {
+        manifestFile = {
+          ngccVersion: NGCC_VERSION,
+          lockFileHash: createHash('md5').update('LOCK FILE CONTENTS').digest('hex'),
+          configFileHash: config.hash,
+          entryPointPaths: []
+        };
+      });
+
+      it('should return null if the base path is not node_modules', () => {
+        fs.ensureDir(_Abs('/project/dist'));
+        fs.writeFile(_Abs('/project/yarn.lock'), 'LOCK FILE CONTENTS');
+        fs.writeFile(
+            _Abs('/project/dist/__ngcc_entry_points__.json'), JSON.stringify(manifestFile));
+        const entryPoints = manifest.readEntryPointsUsingManifest(_Abs('/project/dist'));
+        expect(entryPoints).toBe(null);
+      });
+
+      it('should return null if there is no package lock-file', () => {
+        fs.ensureDir(_Abs('/project/node_modules'));
+        fs.writeFile(
+            _Abs('/project/node_modules/__ngcc_entry_points__.json'), JSON.stringify(manifestFile));
+        const entryPoints = manifest.readEntryPointsUsingManifest(_Abs('/project/node_modules'));
+        expect(entryPoints).toBe(null);
+      });
+
+      it('should return null if there is no manifest file', () => {
+        fs.ensureDir(_Abs('/project/node_modules'));
+        fs.writeFile(_Abs('/project/yarn.lock'), 'LOCK FILE CONTENTS');
+        const entryPoints = manifest.readEntryPointsUsingManifest(_Abs('/project/node_modules'));
+        expect(entryPoints).toBe(null);
+      });
+
+      it('should return null if the ngcc version does not match', () => {
+        fs.ensureDir(_Abs('/project/node_modules'));
+        fs.writeFile(_Abs('/project/yarn.lock'), 'LOCK FILE CONTENTS');
+        manifestFile.ngccVersion = 'bad-version';
+        fs.writeFile(
+            _Abs('/project/node_modules/__ngcc_entry_points__.json'), JSON.stringify(manifestFile));
+        const entryPoints = manifest.readEntryPointsUsingManifest(_Abs('/project/node_modules'));
+        expect(entryPoints).toBe(null);
+      });
+
+      it('should return null if the config hash does not match', () => {
+        fs.ensureDir(_Abs('/project/node_modules'));
+        fs.writeFile(_Abs('/project/yarn.lock'), 'LOCK FILE CONTENTS');
+        manifestFile.configFileHash = 'bad-hash';
+        fs.writeFile(
+            _Abs('/project/node_modules/__ngcc_entry_points__.json'), JSON.stringify(manifestFile));
+        const entryPoints = manifest.readEntryPointsUsingManifest(_Abs('/project/node_modules'));
+        expect(entryPoints).toBe(null);
+      });
+
+      ['yarn.lock', 'package-lock.json'].forEach(packageLockFilePath => {
+        it('should return null if the lockfile hash does not match', () => {
+          fs.ensureDir(_Abs('/project/node_modules'));
+          fs.writeFile(_Abs(`/project/${packageLockFilePath}`), 'LOCK FILE CONTENTS');
+          manifestFile.lockFileHash = 'bad-hash';
+          fs.writeFile(
+              _Abs('/project/node_modules/__ngcc_entry_points__.json'),
+              JSON.stringify(manifestFile));
+          const entryPoints = manifest.readEntryPointsUsingManifest(_Abs('/project/node_modules'));
+          expect(entryPoints).toBe(null);
+        });
+
+        it('should return an array of entry-points if all the checks pass', () => {
+          fs.ensureDir(_Abs('/project/node_modules'));
+          fs.writeFile(_Abs(`/project/${packageLockFilePath}`), 'LOCK FILE CONTENTS');
+          fs.writeFile(
+              _Abs('/project/node_modules/__ngcc_entry_points__.json'),
+              JSON.stringify(manifestFile));
+          const entryPoints = manifest.readEntryPointsUsingManifest(_Abs('/project/node_modules'));
+          expect(entryPoints).toEqual([]);
+        });
+      });
+
+      it('should read in all the entry-point info', () => {
+        fs.ensureDir(_Abs('/project/node_modules'));
+        fs.writeFile(_Abs('/project/yarn.lock'), 'LOCK FILE CONTENTS');
+        loadTestFiles([
+          {
+            name: _Abs('/project/node_modules/some_package/valid_entry_point/package.json'),
+            contents: createPackageJson('valid_entry_point')
+          },
+          {
+            name: _Abs(
+                '/project/node_modules/some_package/valid_entry_point/valid_entry_point.metadata.json'),
+            contents: 'some meta data'
+          },
+        ]);
+        manifestFile.entryPointPaths.push([
+          _Abs('/project/node_modules/some_package'),
+          _Abs('/project/node_modules/some_package/valid_entry_point')
+        ]);
+        fs.writeFile(
+            _Abs('/project/node_modules/__ngcc_entry_points__.json'), JSON.stringify(manifestFile));
+        const entryPoints = manifest.readEntryPointsUsingManifest(_Abs('/project/node_modules'));
+        expect(entryPoints).toEqual([{
+          name: 'some_package/valid_entry_point', packageJson: jasmine.any(Object),
+              package: _Abs('/project/node_modules/some_package'),
+              path: _Abs('/project/node_modules/some_package/valid_entry_point'),
+              typings: _Abs(
+                  '/project/node_modules/some_package/valid_entry_point/valid_entry_point.d.ts'),
+              compiledByAngular: true, ignoreMissingDependencies: false,
+              generateDeepReexports: false,
+        } as any]);
+      });
+
+      it('should return null if any of the entry-points are not valid', () => {
+        fs.ensureDir(_Abs('/project/node_modules'));
+        fs.writeFile(_Abs('/project/yarn.lock'), 'LOCK FILE CONTENTS');
+        manifestFile.entryPointPaths.push([
+          _Abs('/project/node_modules/some_package'),
+          _Abs('/project/node_modules/some_package/valid_entry_point')
+        ]);
+        fs.writeFile(
+            _Abs('/project/node_modules/__ngcc_entry_points__.json'), JSON.stringify(manifestFile));
+        const entryPoints = manifest.readEntryPointsUsingManifest(_Abs('/project/node_modules'));
+        expect(entryPoints).toEqual(null);
+      });
+    });
+
+    describe('writeEntryPointManifest()', () => {
+      it('should do nothing if there is no package lock-file', () => {
+        manifest.writeEntryPointManifest(_Abs('/project/node_modules'), []);
+        expect(fs.exists(_Abs('/project/node_modules/__ngcc_entry_points__.json'))).toBe(false);
+      });
+
+      it('should write an __ngcc_entry_points__.json file below the base path if there is a yarn.lock file',
+         () => {
+           fs.writeFile(_Abs('/project/yarn.lock'), 'LOCK FILE CONTENTS');
+           manifest.writeEntryPointManifest(_Abs('/project/node_modules'), []);
+           expect(fs.exists(_Abs('/project/node_modules/__ngcc_entry_points__.json'))).toBe(true);
+         });
+
+      it('should write an __ngcc_entry_points__.json file below the base path if there is a package-lock.json file',
+         () => {
+           fs.writeFile(_Abs('/project/package-lock.json'), 'LOCK FILE CONTENTS');
+           manifest.writeEntryPointManifest(_Abs('/project/node_modules'), []);
+           expect(fs.exists(_Abs('/project/node_modules/__ngcc_entry_points__.json'))).toBe(true);
+         });
+
+      it('should write the ngcc version', () => {
+        fs.writeFile(_Abs('/project/yarn.lock'), 'LOCK FILE CONTENTS');
+        manifest.writeEntryPointManifest(_Abs('/project/node_modules'), []);
+        const file: EntryPointManifestFile =
+            JSON.parse(fs.readFile(_Abs('/project/node_modules/__ngcc_entry_points__.json')));
+        expect(file.ngccVersion).toEqual(NGCC_VERSION);
+      });
+
+      it('should write a hash of the yarn.lock file', () => {
+        fs.writeFile(_Abs('/project/yarn.lock'), 'LOCK FILE CONTENTS');
+        manifest.writeEntryPointManifest(_Abs('/project/node_modules'), []);
+        const file: EntryPointManifestFile =
+            JSON.parse(fs.readFile(_Abs('/project/node_modules/__ngcc_entry_points__.json')));
+        expect(file.lockFileHash)
+            .toEqual(createHash('md5').update('LOCK FILE CONTENTS').digest('hex'));
+      });
+
+      it('should write a hash of the package-lock.json file', () => {
+        fs.writeFile(_Abs('/project/package-lock.json'), 'LOCK FILE CONTENTS');
+        manifest.writeEntryPointManifest(_Abs('/project/node_modules'), []);
+        const file: EntryPointManifestFile =
+            JSON.parse(fs.readFile(_Abs('/project/node_modules/__ngcc_entry_points__.json')));
+        expect(file.lockFileHash)
+            .toEqual(createHash('md5').update('LOCK FILE CONTENTS').digest('hex'));
+      });
+
+      it('should write a hash of the project config', () => {
+        fs.writeFile(_Abs('/project/package-lock.json'), 'LOCK FILE CONTENTS');
+        manifest.writeEntryPointManifest(_Abs('/project/node_modules'), []);
+        const file: EntryPointManifestFile =
+            JSON.parse(fs.readFile(_Abs('/project/node_modules/__ngcc_entry_points__.json')));
+        expect(file.configFileHash).toEqual(config.hash);
+      });
+
+      it('should write the package path and entry-point path of each entry-point provided', () => {
+        fs.writeFile(_Abs('/project/package-lock.json'), 'LOCK FILE CONTENTS');
+        const entryPoint1 = {
+          package: _Abs('/project/node_modules/package-1/'),
+          path: _Abs('/project/node_modules/package-1/'),
+        } as unknown as EntryPoint;
+        const entryPoint2 = {
+          package: _Abs('/project/node_modules/package-2/'),
+          path: _Abs('/project/node_modules/package-2/entry-point'),
+        } as unknown as EntryPoint;
+        manifest.writeEntryPointManifest(_Abs('/project/node_modules'), [entryPoint1, entryPoint2]);
+        const file: EntryPointManifestFile =
+            JSON.parse(fs.readFile(_Abs('/project/node_modules/__ngcc_entry_points__.json')));
+        expect(file.entryPointPaths).toEqual([
+          [
+            _Abs('/project/node_modules/package-1/'),
+            _Abs('/project/node_modules/package-1/'),
+          ],
+          [
+            _Abs('/project/node_modules/package-2/'),
+            _Abs('/project/node_modules/package-2/entry-point'),
+          ]
+        ]);
+      });
+    });
+  });
+});

packages/compiler-cli/ngcc/test/packages/entry_point_spec.ts

@@ -466,29 +466,29 @@ runInEachFileSystem(() => {
       expect(getEntryPointFormat(fs, entryPoint, 'main')).toBe('umd');
     });
   });
-
-  function createPackageJson(
-      packageName: string,
-      {excludes, typingsProp = 'typings', typingsIsArray}:
-          {excludes?: string[], typingsProp?: string, typingsIsArray?: boolean} = {}): string {
-    const packageJson: any = {
-      name: `some_package/${packageName}`,
-      [typingsProp]: typingsIsArray ? [`./${packageName}.d.ts`] : `./${packageName}.d.ts`,
-      fesm2015: `./fesm2015/${packageName}.js`,
-      esm2015: `./esm2015/${packageName}.js`,
-      es2015: `./es2015/${packageName}.js`,
-      fesm5: `./fesm5/${packageName}.js`,
-      esm5: `./esm5/${packageName}.js`,
-      main: `./bundles/${packageName}/index.js`,
-      module: './index.js',
-    };
-    if (excludes) {
-      excludes.forEach(exclude => delete packageJson[exclude]);
-    }
-    return JSON.stringify(packageJson);
-  }
 });
 
+export function createPackageJson(
+    packageName: string,
+    {excludes, typingsProp = 'typings', typingsIsArray}:
+        {excludes?: string[], typingsProp?: string, typingsIsArray?: boolean} = {}): string {
+  const packageJson: any = {
+    name: `some_package/${packageName}`,
+    [typingsProp]: typingsIsArray ? [`./${packageName}.d.ts`] : `./${packageName}.d.ts`,
+    fesm2015: `./fesm2015/${packageName}.js`,
+    esm2015: `./esm2015/${packageName}.js`,
+    es2015: `./es2015/${packageName}.js`,
+    fesm5: `./fesm5/${packageName}.js`,
+    esm5: `./esm5/${packageName}.js`,
+    main: `./bundles/${packageName}/index.js`,
+    module: './index.js',
+  };
+  if (excludes) {
+    excludes.forEach(exclude => delete packageJson[exclude]);
+  }
+  return JSON.stringify(packageJson);
+}
+
 export function loadPackageJson(fs: FileSystem, packagePath: string) {
   return JSON.parse(fs.readFile(fs.resolve(packagePath + '/package.json')));
 }