diff --git a/packages/compiler-cli/src/ngtsc/sourcemaps/index.ts b/packages/compiler-cli/src/ngtsc/sourcemaps/index.ts index 583e3b1a0c..c34623e749 100644 --- a/packages/compiler-cli/src/ngtsc/sourcemaps/index.ts +++ b/packages/compiler-cli/src/ngtsc/sourcemaps/index.ts @@ -7,4 +7,4 @@ */ export {RawSourceMap} from './src/raw_source_map'; export {Mapping, SourceFile} from './src/source_file'; -export {SourceFileLoader} from './src/source_file_loader'; +export {MapAndPath, SourceFileLoader} from './src/source_file_loader'; diff --git a/packages/compiler-cli/src/ngtsc/sourcemaps/src/source_file_loader.ts b/packages/compiler-cli/src/ngtsc/sourcemaps/src/source_file_loader.ts index 7017144bb6..cc2b45f517 100644 --- a/packages/compiler-cli/src/ngtsc/sourcemaps/src/source_file_loader.ts +++ b/packages/compiler-cli/src/ngtsc/sourcemaps/src/source_file_loader.ts @@ -208,7 +208,7 @@ export class SourceFileLoader { } /** A small helper structure that is returned from `loadSourceMap()`. */ -interface MapAndPath { +export interface MapAndPath { /** The path to the source map if it was external or `null` if it was inline. */ mapPath: AbsoluteFsPath|null; /** The raw source map itself. */ diff --git a/packages/compiler-cli/test/compliance/README.md b/packages/compiler-cli/test/compliance/README.md index f7124d851a..acfa605089 100644 --- a/packages/compiler-cli/test/compliance/README.md +++ b/packages/compiler-cli/test/compliance/README.md @@ -126,6 +126,28 @@ are intelligently matched to check whether they are equivalent. `__i18nMsg__('message string', [ ['placeholder', 'pair] ], { meta: 'properties'})`. * Attribute markers - for example: `__AttributeMarker.Bindings__`. +### Source-map checks + +To check a mapping, add a `// SOURCE:` comment to the end of a line in an expectation file: + +``` + // SOURCE: "" +``` + +The generated code, stripped of the `// SOURCE: ` comment, will still be checked as normal by the +`expectEmit()` helper. But, prior to that, the source-map segments are checked to ensure that there +is a mapping from `` to `` found in the file at ``. + +Note: + +* The source-url should be absolute, with the directory containing the TEST_CASES.json file assumed + to be `/`. +* Whitespace is important and will be included when comparing the segments. +* There is a single space character between each part of the line. +* Newlines within a mapping must be escaped since the mapping and comment must all appear on a + single line of this file. + + ## Running tests The simplest way to run all the compliance tests is: diff --git a/packages/compiler-cli/test/compliance/linked/BUILD.bazel b/packages/compiler-cli/test/compliance/linked/BUILD.bazel index 304127218b..ea2c513b98 100644 --- a/packages/compiler-cli/test/compliance/linked/BUILD.bazel +++ b/packages/compiler-cli/test/compliance/linked/BUILD.bazel @@ -7,6 +7,8 @@ ts_library( deps = [ "//packages/compiler-cli/linker/babel", "//packages/compiler-cli/src/ngtsc/file_system", + "//packages/compiler-cli/src/ngtsc/logging", + "//packages/compiler-cli/src/ngtsc/sourcemaps", "//packages/compiler-cli/test/compliance/test_helpers", "@npm//@types/babel__core", ], diff --git a/packages/compiler-cli/test/compliance/linked/linked_compile_spec.ts b/packages/compiler-cli/test/compliance/linked/linked_compile_spec.ts index d5cdbb80cf..2994347cdc 100644 --- a/packages/compiler-cli/test/compliance/linked/linked_compile_spec.ts +++ b/packages/compiler-cli/test/compliance/linked/linked_compile_spec.ts @@ -9,6 +9,8 @@ import {PluginObj, transformSync} from '@babel/core'; import {createEs2015LinkerPlugin} from '../../../linker/babel'; import {AbsoluteFsPath, FileSystem} from '../../../src/ngtsc/file_system'; +import {ConsoleLogger, LogLevel} from '../../../src/ngtsc/logging'; +import {MapAndPath, RawSourceMap, SourceFileLoader} from '../../../src/ngtsc/sourcemaps'; import {CompileResult, getBuildOutputDirectory} from '../test_helpers/compile_test'; import {ComplianceTest} from '../test_helpers/get_compliance_tests'; import {parseGoldenPartial} from '../test_helpers/golden_partials'; @@ -23,6 +25,8 @@ runTests('linked compile', linkPartials); * @param test The compliance test whose partials will be linked. */ function linkPartials(fs: FileSystem, test: ComplianceTest): CompileResult { + const logger = new ConsoleLogger(LogLevel.debug); + const loader = new SourceFileLoader(fs, logger, {}); const builtDirectory = getBuildOutputDirectory(fs); const linkerPlugin = createEs2015LinkerPlugin({ // By default we don't render legacy message ids in compliance tests. @@ -38,11 +42,30 @@ function linkPartials(fs: FileSystem, test: ComplianceTest): CompileResult { test.relativePath}.golden.update`); } const partialFile = fs.readFile(goldenPartialPath); - const partials = parseGoldenPartial(partialFile).filter(f => f.path.endsWith('.js')); - for (const partial of partials) { - const linkedSource = - applyLinker({fileName: partial.path, source: partial.content}, linkerPlugin); - safeWrite(fs, fs.resolve(builtDirectory, partial.path), linkedSource); + const partialFiles = parseGoldenPartial(partialFile); + + partialFiles.forEach(f => safeWrite(fs, fs.resolve(builtDirectory, f.path), f.content)); + + for (const expectation of test.expectations) { + for (const {generated: fileName} of expectation.files) { + const partialPath = fs.resolve(builtDirectory, fileName); + if (!fs.exists(partialPath)) { + continue; + } + const source = fs.readFile(partialPath); + const sourceMapPath = fs.resolve(partialPath + '.map'); + const sourceMap = + fs.exists(sourceMapPath) ? JSON.parse(fs.readFile(sourceMapPath)) : undefined; + const {linkedSource, linkedSourceMap} = + applyLinker({fileName, source, sourceMap}, linkerPlugin); + + if (linkedSourceMap !== undefined) { + const mapAndPath: MapAndPath = {map: linkedSourceMap, mapPath: sourceMapPath}; + const sourceFile = loader.loadSourceFile(partialPath, linkedSource, mapAndPath); + safeWrite(fs, sourceMapPath, JSON.stringify(sourceFile.renderFlattenedSourceMap())); + } + safeWrite(fs, partialPath, linkedSource); + } } return {emittedFiles: [], errors: []}; } @@ -56,12 +79,15 @@ function linkPartials(fs: FileSystem, test: ComplianceTest): CompileResult { * @param linkerPlugin The linker plugin to apply. * @returns The file's source content, which has been transformed using the linker if necessary. */ -function applyLinker(file: {fileName: string; source: string}, linkerPlugin: PluginObj): string { +function applyLinker( + file: {fileName: string; source: string, sourceMap: RawSourceMap | undefined}, + linkerPlugin: PluginObj): {linkedSource: string, linkedSourceMap: RawSourceMap|undefined} { if (!file.fileName.endsWith('.js')) { - return file.source; + return {linkedSource: file.source, linkedSourceMap: file.sourceMap}; } const result = transformSync(file.source, { filename: file.fileName, + sourceMaps: !!file.sourceMap, plugins: [linkerPlugin], parserOpts: {sourceType: 'unambiguous'}, }); @@ -71,7 +97,7 @@ function applyLinker(file: {fileName: string; source: string}, linkerPlugin: Plu if (result.code == null) { throw fail('Babel transform result does not have any code'); } - return result.code; + return {linkedSource: result.code, linkedSourceMap: result.map || undefined}; } /** diff --git a/packages/compiler-cli/test/compliance/test_helpers/BUILD.bazel b/packages/compiler-cli/test/compliance/test_helpers/BUILD.bazel index 2f6e61ad85..f7db924eca 100644 --- a/packages/compiler-cli/test/compliance/test_helpers/BUILD.bazel +++ b/packages/compiler-cli/test/compliance/test_helpers/BUILD.bazel @@ -15,6 +15,8 @@ ts_library( "//packages/compiler-cli", "//packages/compiler-cli/src/ngtsc/file_system", "//packages/compiler-cli/src/ngtsc/file_system/testing", + "//packages/compiler-cli/src/ngtsc/logging", + "//packages/compiler-cli/src/ngtsc/sourcemaps", "//packages/compiler-cli/src/ngtsc/testing", "@npm//typescript", ], diff --git a/packages/compiler-cli/test/compliance/test_helpers/check_expectations.ts b/packages/compiler-cli/test/compliance/test_helpers/check_expectations.ts index 80e576c155..47e807a1aa 100644 --- a/packages/compiler-cli/test/compliance/test_helpers/check_expectations.ts +++ b/packages/compiler-cli/test/compliance/test_helpers/check_expectations.ts @@ -14,6 +14,7 @@ import {replaceMacros} from './expected_file_macros'; import {verifyUniqueFunctions} from './function_checks'; import {ExpectedFile, ExtraCheck} from './get_compliance_tests'; import {verifyPlaceholdersIntegrity, verifyUniqueConsts} from './i18n_checks'; +import {checkMappings} from './sourcemap_helpers'; type ExtraCheckFunction = (generated: string, ...extraArgs: any[]) => boolean; const EXTRA_CHECK_FUNCTIONS: Record = { @@ -54,8 +55,10 @@ export function checkExpectations( error.stack = ''; throw error; } - const expected = replaceMacros(fs.readFile(expectedPath)); const generated = fs.readFile(generatedPath); + let expected = fs.readFile(expectedPath); + expected = replaceMacros(expected); + expected = checkMappings(fs, generated, generatedPath, expected); expectEmit( generated, expected, diff --git a/packages/compiler-cli/test/compliance/test_helpers/sourcemap_helpers.ts b/packages/compiler-cli/test/compliance/test_helpers/sourcemap_helpers.ts new file mode 100644 index 0000000000..1ed43c393e --- /dev/null +++ b/packages/compiler-cli/test/compliance/test_helpers/sourcemap_helpers.ts @@ -0,0 +1,212 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {AbsoluteFsPath, FileSystem} from '../../../src/ngtsc/file_system'; +import {ConsoleLogger, LogLevel} from '../../../src/ngtsc/logging'; +import {SourceFileLoader} from '../../../src/ngtsc/sourcemaps'; + +/** + * Check the source-mappings of the generated source file against mappings stored in the expected + * source file. + * + * The source-mappings are encoded into the expected source file in the form of an end-of-line + * comment that has the following syntax: + * + * ``` + * // SOURCE: "" + * ``` + * + * The `path/to/original` path will be absolute within the mock file-system, where the root is the + * directory containing the `TEST_CASES.json` file. The `generated code` and the `original source` + * are not trimmed of whitespace - but there is a single space after the generated and a single + * space before the original source. + * + * @param fs The test file-system where the source, generated and expected files are stored. + * @param generated The content of the generated source file. + * @param generatedPath The absolute path, within the test file-system, of the generated source + * file. + * @param expectedSource The content of the expected source file, containing mapping information. + * @returns The content of the expected source file, stripped of the mapping information. + */ +export function checkMappings( + fs: FileSystem, generated: string, generatedPath: AbsoluteFsPath, + expectedSource: string): string { + const actualMappings = getMappedSegments(fs, generatedPath, generated); + + const {expected, mappings} = extractMappings(fs, expectedSource); + + const failures: string[] = []; + for (const expectedMapping of mappings) { + const failure = checkMapping(actualMappings, expectedMapping); + if (failure !== null) { + failures.push(failure); + } + } + + if (failures.length > 0) { + throw new Error( + `When checking mappings for ${generatedPath}...\n\n` + + `${failures.join('\n\n')}\n\n` + + `All the mappings:\n\n${dumpMappings(actualMappings)}`); + } + + return expected; +} + +/** + * A mapping of a segment of generated text to a segment of source text. + */ +interface SegmentMapping { + /** The generated text in this segment. */ + generated: string; + /** The source text in this segment. */ + source: string; + /** The URL of the source file for this segment. */ + sourceUrl: string; +} + +/** + * Extract the source-map information (encoded in comments - see `checkMappings()`) from the given + * `expected` source content, returning both the `mappings` and the `expected` source code, stripped + * of the source-mapping comments. + * + * @param expected The content of the expected file containing source-map information. + */ +function extractMappings( + fs: FileSystem, expected: string): {expected: string, mappings: SegmentMapping[]} { + const mappings: SegmentMapping[] = []; + // capture and remove source mapping info + expected = expected.replace( + /^(.*?) \/\/ SOURCE: "([^"]*?)" (.*?)$/gm, + (_, rawGenerated: string, rawSourceUrl: string, rawSource: string) => { + // Since segments need to appear on a single line in the expected file, any newlines in the + // segment being checked must be escaped in the expected file and then unescaped here before + // being checked. + const generated = unescape(rawGenerated); + const source = unescape(rawSource); + const sourceUrl = fs.resolve(rawSourceUrl); + + mappings.push({generated, sourceUrl, source}); + return generated; + }); + return {expected, mappings}; +} + +function unescape(str: string): string { + const replacements: Record = {'\\n': '\n', '\\\\': '\\'}; + return str.replace(/\\[n\\]/g, match => replacements[match]); +} + +/** + * Process a generated file to extract human understandable segment mappings. + * + * These mappings are easier to compare in unit tests than the raw SourceMap mappings. + * + * @param fs the test file-system that holds the source and generated files. + * @param generatedPath The path of the generated file to process. + * @param generatedContents The contents of the generated file to process. + * @returns An array of segment mappings for each mapped segment in the given generated file. An + * empty array is returned if there is no source-map file found. + */ +function getMappedSegments( + fs: FileSystem, generatedPath: AbsoluteFsPath, generatedContents: string): SegmentMapping[] { + const logger = new ConsoleLogger(LogLevel.debug); + const loader = new SourceFileLoader(fs, logger, {}); + const generatedFile = loader.loadSourceFile(generatedPath, generatedContents); + if (generatedFile === null) { + return []; + } + + const segments: SegmentMapping[] = []; + for (let i = 0; i < generatedFile.flattenedMappings.length - 1; i++) { + const mapping = generatedFile.flattenedMappings[i]; + const generatedStart = mapping.generatedSegment; + const generatedEnd = generatedFile.flattenedMappings[i + 1].generatedSegment; + const originalFile = mapping.originalSource; + const originalStart = mapping.originalSegment; + let originalEnd = originalStart.next; + // Skip until we find an end segment that is after the start segment + while (originalEnd !== undefined && originalEnd.next !== originalEnd && + originalEnd.position === originalStart.position) { + originalEnd = originalEnd.next; + } + if (originalEnd === undefined || originalEnd.next === originalEnd) { + continue; + } + + const segment = { + generated: generatedFile.contents.substring(generatedStart.position, generatedEnd.position), + source: originalFile.contents.substring(originalStart.position, originalEnd!.position), + sourceUrl: originalFile.sourcePath + }; + segments.push(segment); + } + + return segments; +} + +/** + * Check that the `expected` segment appears in the collection of `mappings`. + * + * @returns An error message if a matching segment cannot be found, or null if it can. + */ +function checkMapping(mappings: SegmentMapping[], expected: SegmentMapping): string|null { + if (mappings.some( + m => m.generated === expected.generated && m.source === expected.source && + m.sourceUrl === expected.sourceUrl)) { + return null; + } + const matchingGenerated = mappings.filter(m => m.generated === expected.generated); + const matchingSource = mappings.filter(m => m.source === expected.source); + + const message = [ + 'Expected mappings to contain the following mapping', + prettyPrintMapping(expected), + ]; + if (matchingGenerated.length > 0) { + message.push(''); + message.push('There are the following mappings that match the generated text:'); + matchingGenerated.forEach(m => message.push(prettyPrintMapping(m))); + } + if (matchingSource.length > 0) { + message.push(''); + message.push('There are the following mappings that match the source text:'); + matchingSource.forEach(m => message.push(prettyPrintMapping(m))); + } + + return message.join('\n'); +} + +function prettyPrintMapping(mapping: SegmentMapping): string { + return [ + '{', + ` generated: ${JSON.stringify(mapping.generated)}`, + ` source : ${JSON.stringify(mapping.source)}`, + ` sourceUrl: ${JSON.stringify(mapping.sourceUrl)}`, + '}', + ].join('\n'); +} + +/** + * Helper function for debugging failed mappings. + * This lays out the segment mappings in the console to make it easier to compare. + */ +function dumpMappings(mappings: SegmentMapping[]): string { + return mappings + .map( + mapping => padValue(mapping.sourceUrl, 20, 0) + ' : ' + + padValue(JSON.stringify(mapping.source), 100, 23) + ' : ' + + JSON.stringify(mapping.generated)) + .join('\n'); +} + +function padValue(value: string, max: number, start: number): string { + const padding = value.length > max ? ('\n' + + ' '.repeat(max + start)) : + ' '.repeat(max - value.length); + return value + padding; +}