honeymoose
/
angular-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(compiler): tidy up interpolation splitting (#39717) 

When parsing for i18n messages, interpolated strings are
split into `Text` and `Placeholder` pieces.  The method that
does this `_visitTextWithInterpolation()` was becoming too
complex. This commit refactors that method along with some
associated functions that it uses.

PR Close #39717
This commit is contained in:
Pete Bacon Darwin 2020-11-17 10:07:54 +00:00 committed by Andrew Kushnir
parent 0462a616c3
commit 969ad329de
4 changed files with 141 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
packages/compiler/src/expression_parser/parser.ts
View File

@ -13,10 +13,14 @@ import {escapeRegExp} from '../util';
import {AbsoluteSourceSpan, AST, AstVisitor, ASTWithSource, Binary, BindingPipe, Chain, Conditional, EmptyExpr, ExpressionBinding, FunctionCall, ImplicitReceiver, Interpolation, KeyedRead, KeyedWrite, LiteralArray, LiteralMap, LiteralMapKey, LiteralPrimitive, MethodCall, NonNullAssert, ParserError, ParseSpan, PrefixNot, PropertyRead, PropertyWrite, Quote, RecursiveAstVisitor, SafeMethodCall, SafePropertyRead, TemplateBinding, TemplateBindingIdentifier, ThisReceiver, Unary, VariableBinding} from './ast'; import {AbsoluteSourceSpan, AST, AstVisitor, ASTWithSource, Binary, BindingPipe, Chain, Conditional, EmptyExpr, ExpressionBinding, FunctionCall, ImplicitReceiver, Interpolation, KeyedRead, KeyedWrite, LiteralArray, LiteralMap, LiteralMapKey, LiteralPrimitive, MethodCall, NonNullAssert, ParserError, ParseSpan, PrefixNot, PropertyRead, PropertyWrite, Quote, RecursiveAstVisitor, SafeMethodCall, SafePropertyRead, TemplateBinding, TemplateBindingIdentifier, ThisReceiver, Unary, VariableBinding} from './ast';
import {EOF, isIdentifier, isQuote, Lexer, Token, TokenType} from './lexer'; import {EOF, isIdentifier, isQuote, Lexer, Token, TokenType} from './lexer';
export interface InterpolationPiece {
text: string;
start: number;
end: number;
}
export class SplitInterpolation { export class SplitInterpolation {
constructor( constructor(
public strings: string[], public stringSpans: {start: number, end: number}[], public strings: InterpolationPiece[], public expressions: InterpolationPiece[],
public expressions: string[], public expressionsSpans: {start: number, end: number}[],
public offsets: number[]) {} public offsets: number[]) {}
} }
@ -48,7 +52,7 @@ export class Parser {
simpleExpressionChecker = SimpleExpressionChecker; simpleExpressionChecker = SimpleExpressionChecker;
parseAction( parseAction(
input: string, location: any, absoluteOffset: number, input: string, location: string, absoluteOffset: number,
interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): ASTWithSource { interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): ASTWithSource {
this._checkNoInterpolation(input, location, interpolationConfig); this._checkNoInterpolation(input, location, interpolationConfig);
const sourceToLex = this._stripComments(input); const sourceToLex = this._stripComments(input);
@ -61,7 +65,7 @@ export class Parser {
} }
parseBinding( parseBinding(
input: string, location: any, absoluteOffset: number, input: string, location: string, absoluteOffset: number,
interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): ASTWithSource { interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): ASTWithSource {
const ast = this._parseBindingAst(input, location, absoluteOffset, interpolationConfig); const ast = this._parseBindingAst(input, location, absoluteOffset, interpolationConfig);
return new ASTWithSource(ast, input, location, absoluteOffset, this.errors); return new ASTWithSource(ast, input, location, absoluteOffset, this.errors);
@ -85,7 +89,7 @@ export class Parser {
return new ASTWithSource(ast, input, location, absoluteOffset, this.errors); return new ASTWithSource(ast, input, location, absoluteOffset, this.errors);
} }
private _reportError(message: string, input: string, errLocation: string, ctxLocation?: any) { private _reportError(message: string, input: string, errLocation: string, ctxLocation?: string) {
this.errors.push(new ParserError(message, input, errLocation, ctxLocation)); this.errors.push(new ParserError(message, input, errLocation, ctxLocation));
} }
@ -109,7 +113,7 @@ export class Parser {
.parseChain(); .parseChain();
} }
private _parseQuote(input: string|null, location: any, absoluteOffset: number): AST|null { private _parseQuote(input: string|null, location: string, absoluteOffset: number): AST|null {
if (input == null) return null; if (input == null) return null;
const prefixSeparatorIndex = input.indexOf(':'); const prefixSeparatorIndex = input.indexOf(':');
if (prefixSeparatorIndex == -1) return null; if (prefixSeparatorIndex == -1) return null;
@ -161,25 +165,27 @@ export class Parser {
} }
parseInterpolation( parseInterpolation(
input: string, location: any, absoluteOffset: number, input: string, location: string, absoluteOffset: number,
interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): ASTWithSource|null { interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): ASTWithSource|null {
const split = this.splitInterpolation(input, location, interpolationConfig); const {strings, expressions, offsets} =
if (split == null) return null; this.splitInterpolation(input, location, interpolationConfig);
if (expressions.length === 0) return null;
const expressions: AST[] = []; const expressionNodes: AST[] = [];
for (let i = 0; i < split.expressions.length; ++i) { for (let i = 0; i < expressions.length; ++i) {
const expressionText = split.expressions[i]; const expressionText = expressions[i].text;
const sourceToLex = this._stripComments(expressionText); const sourceToLex = this._stripComments(expressionText);
const tokens = this._lexer.tokenize(sourceToLex); const tokens = this._lexer.tokenize(sourceToLex);
const ast = new _ParseAST( const ast = new _ParseAST(
input, location, absoluteOffset, tokens, sourceToLex.length, false, input, location, absoluteOffset, tokens, sourceToLex.length, false,
this.errors, split.offsets[i] + (expressionText.length - sourceToLex.length)) this.errors, offsets[i] + (expressionText.length - sourceToLex.length))
.parseChain(); .parseChain();
expressions.push(ast); expressionNodes.push(ast);
} }
return this.createInterpolationAst(split.strings, expressions, input, location, absoluteOffset); return this.createInterpolationAst(
strings.map(s => s.text), expressionNodes, input, location, absoluteOffset);
} }
/** /**
@ -187,7 +193,7 @@ export class Parser {
* element that would normally appear within the interpolation prefix and suffix (`{{` and `}}`). * element that would normally appear within the interpolation prefix and suffix (`{{` and `}}`).
* This is used for parsing the switch expression in ICUs. * This is used for parsing the switch expression in ICUs.
*/ */
parseInterpolationExpression(expression: string, location: any, absoluteOffset: number): parseInterpolationExpression(expression: string, location: string, absoluteOffset: number):
ASTWithSource { ASTWithSource {
const sourceToLex = this._stripComments(expression); const sourceToLex = this._stripComments(expression);
const tokens = this._lexer.tokenize(sourceToLex); const tokens = this._lexer.tokenize(sourceToLex);
@ -217,13 +223,10 @@ export class Parser {
*/ */
splitInterpolation( splitInterpolation(
input: string, location: string, input: string, location: string,
interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): SplitInterpolation interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): SplitInterpolation {
|null { const strings: InterpolationPiece[] = [];
const strings: string[] = []; const expressions: InterpolationPiece[] = [];
const expressions: string[] = [];
const offsets: number[] = []; const offsets: number[] = [];
const stringSpans: {start: number, end: number}[] = [];
const expressionSpans: {start: number, end: number}[] = [];
let i = 0; let i = 0;
let atInterpolation = false; let atInterpolation = false;
let extendLastString = false; let extendLastString = false;
@ -236,9 +239,8 @@ export class Parser {
if (i === -1) { if (i === -1) {
i = input.length; i = input.length;
} }
const part = input.substring(start, i); const text = input.substring(start, i);
strings.push(part); strings.push({text, start, end: i});
stringSpans.push({start, end: i});
atInterpolation = true; atInterpolation = true;
} else { } else {
@ -255,17 +257,16 @@ export class Parser {
} }
const fullEnd = exprEnd + interpEnd.length; const fullEnd = exprEnd + interpEnd.length;
const part = input.substring(exprStart, exprEnd); const text = input.substring(exprStart, exprEnd);
if (part.trim().length > 0) { if (text.trim().length > 0) {
expressions.push(part); expressions.push({text, start: fullStart, end: fullEnd});
} else { } else {
this._reportError( this._reportError(
'Blank expressions are not allowed in interpolated strings', input, 'Blank expressions are not allowed in interpolated strings', input,
`at column ${i} in`, location); `at column ${i} in`, location);
expressions.push('$implicit'); expressions.push({text: '$implicit', start: fullStart, end: fullEnd});
} }
offsets.push(exprStart); offsets.push(exprStart);
expressionSpans.push({start: fullStart, end: fullEnd});
i = fullEnd; i = fullEnd;
atInterpolation = false; atInterpolation = false;
@ -274,19 +275,18 @@ export class Parser {
if (!atInterpolation) { if (!atInterpolation) {
// If we are now at a text section, add the remaining content as a raw string. // If we are now at a text section, add the remaining content as a raw string.
if (extendLastString) { if (extendLastString) {
strings[strings.length - 1] += input.substring(i); const piece = strings[strings.length - 1];
stringSpans[stringSpans.length - 1].end = input.length; piece.text += input.substring(i);
piece.end = input.length;
} else { } else {
strings.push(input.substring(i)); strings.push({text: input.substring(i), start: i, end: input.length});
stringSpans.push({start: i, end: input.length});
} }
} }
return expressions.length === 0 ? return new SplitInterpolation(strings, expressions, offsets);
null :
new SplitInterpolation(strings, stringSpans, expressions, expressionSpans, offsets);
} }
wrapLiteralPrimitive(input: string|null, location: any, absoluteOffset: number): ASTWithSource { wrapLiteralPrimitive(input: string|null, location: string, absoluteOffset: number):
ASTWithSource {
const span = new ParseSpan(0, input == null ? 0 : input.length); const span = new ParseSpan(0, input == null ? 0 : input.length);
return new ASTWithSource( return new ASTWithSource(
new LiteralPrimitive(span, span.toAbsolute(absoluteOffset), input), input, location, new LiteralPrimitive(span, span.toAbsolute(absoluteOffset), input), input, location,
@ -316,7 +316,7 @@ export class Parser {
} }
private _checkNoInterpolation( private _checkNoInterpolation(
input: string, location: any, interpolationConfig: InterpolationConfig): void { input: string, location: string, interpolationConfig: InterpolationConfig): void {
const regexp = _getInterpolateRegExp(interpolationConfig); const regexp = _getInterpolateRegExp(interpolationConfig);
const parts = input.split(regexp); const parts = input.split(regexp);
if (parts.length > 1) { if (parts.length > 1) {
@ -374,7 +374,7 @@ export class _ParseAST {
index: number = 0; index: number = 0;
constructor( constructor(
public input: string, public location: any, public absoluteOffset: number, public input: string, public location: string, public absoluteOffset: number,
public tokens: Token[], public inputLength: number, public parseAction: boolean, public tokens: Token[], public inputLength: number, public parseAction: boolean,
private errors: ParserError[], private offset: number) {} private errors: ParserError[], private offset: number) {}

152
packages/compiler/src/i18n/i18n_parser.ts
View File

@ -7,7 +7,7 @@
*/ */
import {Lexer as ExpressionLexer} from '../expression_parser/lexer'; import {Lexer as ExpressionLexer} from '../expression_parser/lexer';
import {Parser as ExpressionParser} from '../expression_parser/parser'; import {InterpolationPiece, Parser as ExpressionParser} from '../expression_parser/parser';
import * as html from '../ml_parser/ast'; import * as html from '../ml_parser/ast';
import {getHtmlTagDefinition} from '../ml_parser/html_tags'; import {getHtmlTagDefinition} from '../ml_parser/html_tags';
import {InterpolationConfig} from '../ml_parser/interpolation_config'; import {InterpolationConfig} from '../ml_parser/interpolation_config';
@ -156,73 +156,110 @@ class _I18nVisitor implements html.Visitor {
throw new Error('Unreachable code'); throw new Error('Unreachable code');
} }
/**
* Split the, potentially interpolated, text up into text and placeholder pieces.
*
* @param text The potentially interpolated string to be split.
* @param sourceSpan The span of the whole of the `text` string.
* @param context The current context of the visitor, used to compute and store placeholders.
* @param previousI18n Any i18n metadata associated with this `text` from a previous pass.
*/
private _visitTextWithInterpolation( private _visitTextWithInterpolation(
text: string, sourceSpan: ParseSourceSpan, context: I18nMessageVisitorContext, text: string, sourceSpan: ParseSourceSpan, context: I18nMessageVisitorContext,
previousI18n: i18n.I18nMeta|undefined): i18n.Node { previousI18n: i18n.I18nMeta|undefined): i18n.Node {
const splitInterpolation = this._expressionParser.splitInterpolation( const {strings, expressions} = this._expressionParser.splitInterpolation(
text, sourceSpan.start.toString(), this._interpolationConfig); text, sourceSpan.start.toString(), this._interpolationConfig);
if (!splitInterpolation) { // No expressions, return a single text.
// No expression, return a single text if (expressions.length === 0) {
return new i18n.Text(text, sourceSpan); return new i18n.Text(text, sourceSpan);
} }
// Return a group of text + expressions // Return a sequence of `Text` and `Placeholder` nodes grouped in a `Container`.
const nodes: i18n.Node[] = []; const nodes: i18n.Node[] = [];
const container = new i18n.Container(nodes, sourceSpan); for (let i = 0; i < strings.length - 1; i++) {
const {start: sDelimiter, end: eDelimiter} = this._interpolationConfig; this._addText(nodes, strings[i], sourceSpan);
this._addPlaceholder(nodes, context, expressions[i], sourceSpan);
for (let i = 0; i < splitInterpolation.strings.length - 1; i++) {
const expression = splitInterpolation.expressions[i];
const baseName = _extractPlaceholderName(expression) || 'INTERPOLATION';
const phName = context.placeholderRegistry.getPlaceholderName(baseName, expression);
if (splitInterpolation.strings[i].length) {
// No need to add empty strings
const stringSpan = getOffsetSourceSpan(sourceSpan, splitInterpolation.stringSpans[i]);
nodes.push(new i18n.Text(splitInterpolation.strings[i], stringSpan));
}
const expressionSpan =
getOffsetSourceSpan(sourceSpan, splitInterpolation.expressionsSpans[i]);
nodes.push(new i18n.Placeholder(expression, phName, expressionSpan));
context.placeholderToContent[phName] = {
text: sDelimiter + expression + eDelimiter,
sourceSpan: expressionSpan,
};
} }
// The last index contains no expression // The last index contains no expression
const lastStringIdx = splitInterpolation.strings.length - 1; this._addText(nodes, strings[strings.length - 1], sourceSpan);
if (splitInterpolation.strings[lastStringIdx].length) {
const stringSpan = // Whitespace removal may have invalidated the interpolation source-spans.
getOffsetSourceSpan(sourceSpan, splitInterpolation.stringSpans[lastStringIdx]); reusePreviousSourceSpans(nodes, previousI18n);
nodes.push(new i18n.Text(splitInterpolation.strings[lastStringIdx], stringSpan));
return new i18n.Container(nodes, sourceSpan);
}
/**
* Create a new `Text` node from the `textPiece` and add it to the `nodes` collection.
*
* @param nodes The nodes to which the created `Text` node should be added.
* @param textPiece The text and relative span information for this `Text` node.
* @param interpolationSpan The span of the whole interpolated text.
*/
private _addText(
nodes: i18n.Node[], textPiece: InterpolationPiece, interpolationSpan: ParseSourceSpan): void {
if (textPiece.text.length > 0) {
// No need to add empty strings
const stringSpan = getOffsetSourceSpan(interpolationSpan, textPiece);
nodes.push(new i18n.Text(textPiece.text, stringSpan));
} }
}
if (previousI18n instanceof i18n.Message) { /**
// The `previousI18n` is an i18n `Message`, so we are processing an `Attribute` with i18n * Create a new `Placeholder` node from the `expression` and add it to the `nodes` collection.
// metadata. The `Message` should consist only of a single `Container` that contains the *
// parts (`Text` and `Placeholder`) to process. * @param nodes The nodes to which the created `Text` node should be added.
assertSingleContainerMessage(previousI18n); * @param context The current context of the visitor, used to compute and store placeholders.
previousI18n = previousI18n.nodes[0]; * @param expression The expression text and relative span information for this `Placeholder`
} * node.
* @param interpolationSpan The span of the whole interpolated text.
if (previousI18n instanceof i18n.Container) { */
// The `previousI18n` is a `Container`, which means that this is a second i18n extraction pass private _addPlaceholder(
// after whitespace has been removed from the AST ndoes. nodes: i18n.Node[], context: I18nMessageVisitorContext, expression: InterpolationPiece,
assertEquivalentNodes(previousI18n.children, nodes); interpolationSpan: ParseSourceSpan): void {
const sourceSpan = getOffsetSourceSpan(interpolationSpan, expression);
// Reuse the source-spans from the first pass. const baseName = extractPlaceholderName(expression.text) || 'INTERPOLATION';
for (let i = 0; i < nodes.length; i++) { const phName = context.placeholderRegistry.getPlaceholderName(baseName, expression.text);
nodes[i].sourceSpan = previousI18n.children[i].sourceSpan; const text = this._interpolationConfig.start + expression.text + this._interpolationConfig.end;
} context.placeholderToContent[phName] = {text, sourceSpan};
} nodes.push(new i18n.Placeholder(expression.text, phName, sourceSpan));
return container;
} }
} }
/**
* Re-use the source-spans from `previousI18n` metadata for the `nodes`.
*
* Whitespace removal can invalidate the source-spans of interpolation nodes, so we
* reuse the source-span stored from a previous pass before the whitespace was removed.
*
* @param nodes The `Text` and `Placeholder` nodes to be processed.
* @param previousI18n Any i18n metadata for these `nodes` stored from a previous pass.
*/
function reusePreviousSourceSpans(nodes: i18n.Node[], previousI18n: i18n.I18nMeta|undefined): void {
if (previousI18n instanceof i18n.Message) {
// The `previousI18n` is an i18n `Message`, so we are processing an `Attribute` with i18n
// metadata. The `Message` should consist only of a single `Container` that contains the
// parts (`Text` and `Placeholder`) to process.
assertSingleContainerMessage(previousI18n);
previousI18n = previousI18n.nodes[0];
}
if (previousI18n instanceof i18n.Container) {
// The `previousI18n` is a `Container`, which means that this is a second i18n extraction pass
// after whitespace has been removed from the AST ndoes.
assertEquivalentNodes(previousI18n.children, nodes);
// Reuse the source-spans from the first pass.
for (let i = 0; i < nodes.length; i++) {
nodes[i].sourceSpan = previousI18n.children[i].sourceSpan;
}
}
}
/**
* Asserts that the `message` contains exactly one `Container` node.
*/
function assertSingleContainerMessage(message: i18n.Message): void { function assertSingleContainerMessage(message: i18n.Message): void {
const nodes = message.nodes; const nodes = message.nodes;
if (nodes.length !== 1 || !(nodes[0] instanceof i18n.Container)) { if (nodes.length !== 1 || !(nodes[0] instanceof i18n.Container)) {
@ -231,6 +268,10 @@ function assertSingleContainerMessage(message: i18n.Message): void {
} }
} }
/**
* Asserts that the `previousNodes` and `node` collections have the same number of elements and
* corresponding elements have the same node type.
*/
function assertEquivalentNodes(previousNodes: i18n.Node[], nodes: i18n.Node[]): void { function assertEquivalentNodes(previousNodes: i18n.Node[], nodes: i18n.Node[]): void {
if (previousNodes.length !== nodes.length) { if (previousNodes.length !== nodes.length) {
throw new Error('The number of i18n message children changed between first and second pass.'); throw new Error('The number of i18n message children changed between first and second pass.');
@ -241,14 +282,17 @@ function assertEquivalentNodes(previousNodes: i18n.Node[], nodes: i18n.Node[]):
} }
} }
/**
* Create a new `ParseSourceSpan` from the `sourceSpan`, offset by the `start` and `end` values.
*/
function getOffsetSourceSpan( function getOffsetSourceSpan(
sourceSpan: ParseSourceSpan, {start, end}: {start: number, end: number}): ParseSourceSpan { sourceSpan: ParseSourceSpan, {start, end}: InterpolationPiece): ParseSourceSpan {
return new ParseSourceSpan(sourceSpan.fullStart.moveBy(start), sourceSpan.fullStart.moveBy(end)); return new ParseSourceSpan(sourceSpan.fullStart.moveBy(start), sourceSpan.fullStart.moveBy(end));
} }
const _CUSTOM_PH_EXP = const _CUSTOM_PH_EXP =
/\/\/[\s\S]*i18n[\s\S]*\([\s\S]*ph[\s\S]*=[\s\S]*("|')([\s\S]*?)\1[\s\S]*\)/g; /\/\/[\s\S]*i18n[\s\S]*\([\s\S]*ph[\s\S]*=[\s\S]*("|')([\s\S]*?)\1[\s\S]*\)/g;
function _extractPlaceholderName(input: string): string { function extractPlaceholderName(input: string): string {
return input.split(_CUSTOM_PH_EXP)[2]; return input.split(_CUSTOM_PH_EXP)[2];
} }

2
packages/compiler/test/expression_parser/ast_spec.ts
View File

@ -12,7 +12,7 @@ import {ImplicitReceiver, MethodCall, PropertyRead} from '@angular/compiler/src/
describe('RecursiveAstVisitor', () => { describe('RecursiveAstVisitor', () => {
it('should visit every node', () => { it('should visit every node', () => {
const parser = new Parser(new Lexer()); const parser = new Parser(new Lexer());
const ast = parser.parseBinding('x.y()', null /* location */, 0 /* absoluteOffset */); const ast = parser.parseBinding('x.y()', '', 0 /* absoluteOffset */);
const visitor = new Visitor(); const visitor = new Visitor();
const path: AST[] = []; const path: AST[] = [];
visitor.visit(ast.ast, path); visitor.visit(ast.ast, path);

6
packages/compiler/test/expression_parser/parser_spec.ts
View File

@ -855,8 +855,7 @@ describe('parser', () => {
it('should support custom interpolation', () => { it('should support custom interpolation', () => {
const parser = new Parser(new Lexer()); const parser = new Parser(new Lexer());
const ast = const ast = parser.parseInterpolation('{% a %}', '', 0, {start: '{%', end: '%}'})!.ast as any;
parser.parseInterpolation('{% a %}', null, 0, {start: '{%', end: '%}'})!.ast as any;
expect(ast.strings).toEqual(['', '']); expect(ast.strings).toEqual(['', '']);
expect(ast.expressions.length).toEqual(1); expect(ast.expressions.length).toEqual(1);
expect(ast.expressions[0].name).toEqual('a'); expect(ast.expressions[0].name).toEqual('a');
@ -978,8 +977,7 @@ describe('parser', () => {
describe('wrapLiteralPrimitive', () => { describe('wrapLiteralPrimitive', () => {
it('should wrap a literal primitive', () => { it('should wrap a literal primitive', () => {
expect(unparse(validate(createParser().wrapLiteralPrimitive('foo', null, 0)))) expect(unparse(validate(createParser().wrapLiteralPrimitive('foo', '', 0)))).toEqual('"foo"');
.toEqual('"foo"');
}); });
}); });