honeymoose
/
angular-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: simplify generation of closure locale file (#42230) 

In the past, the closure file has been generated so that all individual
locale files were imported individually. This resulted in a huge
slow-down in g3 due to the large amount of imports.

With 90bd984ff7 this changed so that we
inline the locale data for the g3 closure locale file. Also the file
only contained data for locales being supported by Closure. For this a
list of locales has been extracted from Closure Compiler, as well as a
list of locale aliases.

This logic is prone to CLDR version updates, and also broke as part of
the Gulp -> Bazel migration where this logic has been slightly modified
but caused issues in G3. e.g. a locale `zh-Hant` was requested in g3,
but the locale data had the name of the alias locale that provided the
data at index zero (which represents the locale name). Note that the
locale names at index zero always could differentiate from the requested
`goog.LOCALE` due to the aliasing logic. This just didn't come up before.

We simplify this logic by generating a `goog.LOCALE` case for all
locales CLDR provides data for. We don't need to bother about aliasing
because with the refactorings to the CLDR generation tool, all locales
are built (which also captures the aliases), and we can generate the locale
file on the fly (which has not been done before).

PR Close #42230
This commit is contained in:
Paul Gschwendtner 2021-06-02 15:50:07 +02:00 committed by Jessica Janiuk
parent 7b288471bf
commit b762820485
4 changed files with 6064 additions and 706 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6590
packages/common/locales/closure-locale.ts
View File

File diff suppressed because it is too large Load Diff

18
packages/common/locales/generate-locales-tool/bin/write-locale-files-to-dist.ts
View File

@ -31,22 +31,8 @@ function main(outputDir: string) {
console.info(`Writing locales to: ${outputDir}`); console.info(`Writing locales to: ${outputDir}`);
// Generate locale files for all locales we have data for. // Generate locale files for all locales we have data for.
cldrData.availableLocales.forEach((locale: string) => { cldrData.availableLocales.forEach(localeData => {
const localeData = cldrData.getLocaleData(locale); const locale = localeData.locale;
// If `cldrjs` is unable to resolve a `bundle` for the current locale, then there is no data
// for this locale, and it should not be generated. This can happen as with older versions of
// CLDR where `availableLocales.json` specifies locales for which no data is available
// (even within the `full` tier packages). See:
// http://cldr.unicode.org/development/development-process/design-proposals/json-packaging.
// TODO(devversion): Remove if we update to CLDR v39 where this seems fixed. Note that this
// worked before in the Gulp tooling without such a check because the `cldr-data-downloader`
// overwrote the `availableLocales` to only capture locales with data.
if (localeData && !(localeData.attributes as any).bundle) {
console.info(`Skipping generation of ${locale} as there is no data.`);
return;
}
const localeFile = generateLocale(locale, localeData, baseCurrencies); const localeFile = generateLocale(locale, localeData, baseCurrencies);
const localeExtraFile = generateLocaleExtra(locale, localeData); const localeExtraFile = generateLocaleExtra(locale, localeData);
const localeGlobalFile = generateLocaleGlobalFile(locale, localeData, baseCurrencies); const localeGlobalFile = generateLocaleGlobalFile(locale, localeData, baseCurrencies);

26
packages/common/locales/generate-locales-tool/cldr-data.ts
View File

@ -40,7 +40,7 @@ export class CldrData {
cldrDataDir = runfiles.resolve('cldr_data'); cldrDataDir = runfiles.resolve('cldr_data');
/** List of all available locales CLDR provides data for. */ /** List of all available locales CLDR provides data for. */
availableLocales: string[]; availableLocales: CldrStatic[];
constructor() { constructor() {
this._loadAndPopulateCldrData(); this._loadAndPopulateCldrData();
@ -53,8 +53,28 @@ export class CldrData {
} }
/** Gets a list of all locales CLDR provides data for. */ /** Gets a list of all locales CLDR provides data for. */
private _getAvailableLocales(): string[] { private _getAvailableLocales(): CldrStatic[] {
return require(`${this.cldrDataDir}/${CLDR_AVAILABLE_LOCALES_PATH}`).availableLocales.full; const allLocales =
require(`${this.cldrDataDir}/${CLDR_AVAILABLE_LOCALES_PATH}`).availableLocales.full;
const localesWithData: CldrStatic[] = [];
for (const localeName of allLocales) {
const localeData = this.getLocaleData(localeName);
// If `cldrjs` is unable to resolve a `bundle` for the current locale, then there is no data
// for this locale, and it should not be generated. This can happen as with older versions of
// CLDR where `availableLocales.json` specifies locales for which no data is available
// (even within the `full` tier packages). See:
// http://cldr.unicode.org/development/development-process/design-proposals/json-packaging.
// TODO(devversion): Remove if we update to CLDR v39 where this seems fixed. Note that this
// worked before in the Gulp tooling without such a check because the `cldr-data-downloader`
// overwrote the `availableLocales` to only capture locales with data.
if (localeData && (localeData.attributes as any).bundle) {
localesWithData.push(localeData);
}
}
return localesWithData;
} }
/** Loads the CLDR data and populates the `cldrjs` library with it. */ /** Loads the CLDR data and populates the `cldrjs` library with it. */

136
packages/common/locales/generate-locales-tool/closure-locale-file.ts
View File

@ -6,126 +6,35 @@
* found in the LICENSE file at https://angular.io/license * found in the LICENSE file at https://angular.io/license
*/ */
import {CldrStatic} from 'cldrjs';
import {CldrData} from './cldr-data'; import {CldrData} from './cldr-data';
import {fileHeader} from './file-header'; import {fileHeader} from './file-header';
import {BaseCurrencies} from './locale-base-currencies'; import {BaseCurrencies} from './locale-base-currencies';
import {generateLocale} from './locale-file'; import {generateLocale} from './locale-file';
/**
* List of locales used by Closure. These locales will be incorporated in the generated
* closure locale file. See:
* https://github.com/google/closure-library/blob/master/closure/goog/i18n/datetimepatterns.js#L2450
*/
const GOOG_LOCALES = [
'af', 'am', 'ar', 'ar-DZ', 'az', 'be', 'bg', 'bn', 'br', 'bs',
'ca', 'chr', 'cs', 'cy', 'da', 'de', 'de-AT', 'de-CH', 'el', 'en-AU',
'en-CA', 'en-GB', 'en-IE', 'en-IN', 'en-SG', 'en-ZA', 'es', 'es-419', 'es-MX', 'es-US',
'et', 'eu', 'fa', 'fi', 'fr', 'fr-CA', 'ga', 'gl', 'gsw', 'gu',
'haw', 'hi', 'hr', 'hu', 'hy', 'in', 'is', 'it', 'iw', 'ja',
'ka', 'kk', 'km', 'kn', 'ko', 'ky', 'ln', 'lo', 'lt', 'lv',
'mk', 'ml', 'mn', 'mo', 'mr', 'ms', 'mt', 'my', 'ne', 'nl',
'no', 'or', 'pa', 'pl', 'pt', 'pt-PT', 'ro', 'ru', 'sh', 'si',
'sk', 'sl', 'sq', 'sr', 'sv', 'sw', 'ta', 'te', 'th', 'tl',
'tr', 'uk', 'ur', 'uz', 'vi', 'zh', 'zh-CN', 'zh-HK', 'zh-TW', 'zu'
];
export function generateClosureLocaleFile(
cldrData: CldrData, baseCurrencies: BaseCurrencies): string {
// locale id aliases to support deprecated locale ids used by closure
// it maps deprecated ids --> new ids
// manually extracted from ./cldr-data/supplemental/aliases.json
// TODO: Consider extracting directly from the CLDR data instead.
const aliases = {
'in': 'id',
'iw': 'he',
'mo': 'ro-MD',
'no': 'nb',
'nb': 'no-NO',
'sh': 'sr-Latn',
'tl': 'fil',
'pt': 'pt-BR',
'zh-CN': 'zh-Hans-CN',
'zh-Hans-CN': 'zh-Hans',
'zh-HK': 'zh-Hant-HK',
'zh-TW': 'zh-Hant-TW',
'zh-Hant-TW': 'zh-Hant',
};
return generateAllLocalesFile(cldrData, GOOG_LOCALES, aliases, baseCurrencies);
}
/** /**
* Generate a file that contains all locale to import for closure. * Generate a file that contains all locale to import for closure.
* Tree shaking will only keep the data for the `goog.LOCALE` locale. * Tree shaking will only keep the data for the `goog.LOCALE` locale.
*/ */
function generateAllLocalesFile( export function generateClosureLocaleFile(cldrData: CldrData, baseCurrencies: BaseCurrencies) {
cldrData: CldrData, locales: string[], aliases: {[name: string]: string}, const locales = cldrData.availableLocales;
baseCurrencies: BaseCurrencies) {
const existingLocalesAliases: {[locale: string]: Set<string>} = {};
const existingLocalesData: {[locale: string]: string} = {};
// for each locale, get the data and the list of equivalent locales function generateLocaleConstant(localeData: CldrStatic): string {
locales.forEach(locale => { const locale = localeData.locale;
const eqLocales = new Set<string>(); const localeNameFormattedForJs = formatLocale(locale);
eqLocales.add(locale); return generateLocale(locale, localeData, baseCurrencies)
if (locale.match(/-/)) { .replace(`${fileHeader}\n`, '')
eqLocales.add(locale.replace(/-/g, '_')); .replace('export default ', `export const locale_${localeNameFormattedForJs} = `)
} .replace('function plural', `function plural_${localeNameFormattedForJs}`)
.replace(/,\s+plural/, `, plural_${localeNameFormattedForJs}`)
.replace(/\s*const u = undefined;\s*/, '');
}
// check for aliases function generateCase(localeName: string) {
const alias = aliases[locale]; return `case '${localeName}':\n` +
if (alias) { `l = locale_${formatLocale(localeName)};\n` +
eqLocales.add(alias); `break;\n`;
if (alias.match(/-/)) {
eqLocales.add(alias.replace(/-/g, '_'));
}
// to avoid duplicated "case" we regroup all locales in the same "case"
// the simplest way to do that is to have alias aliases
// e.g. 'no' --> 'nb', 'nb' --> 'no-NO'
// which means that we'll have 'no', 'nb' and 'no-NO' in the same "case"
const aliasKeys = Object.keys(aliases);
for (let i = 0; i < aliasKeys.length; i++) {
const aliasValue = aliases[alias];
if (aliasKeys.indexOf(alias) !== -1 && !eqLocales.has(aliasValue)) {
eqLocales.add(aliasValue);
if (aliasValue.match(/-/)) {
eqLocales.add(aliasValue.replace(/-/g, '_'));
}
}
}
}
const localeNameForData = aliases[locale] ?? locale;
const localeData = cldrData.getLocaleData(localeNameForData);
const localeName = formatLocale(locale);
existingLocalesData[locale] =
generateLocale(localeNameForData, localeData, baseCurrencies)
.replace(`${fileHeader}\n`, '')
.replace('export default ', `export const locale_${localeName} = `)
.replace('function plural', `function plural_${localeName}`)
.replace(/,\s+plural/, `, plural_${localeName}`)
.replace(/\s*const u = undefined;\s*/, '');
existingLocalesAliases[locale] = eqLocales;
});
function generateCases(locale: string) {
let str = '';
let locales: string[] = [];
const eqLocales = existingLocalesAliases[locale];
eqLocales.forEach(l => {
str += `case '${l}':\n`;
locales.push(`'${l}'`);
});
let localesStr = '[' + locales.join(',') + ']';
str += ` l = locale_${formatLocale(locale)};
locales = ${localesStr};
break;\n`;
return str;
} }
return `${fileHeader} return `${fileHeader}
@ -134,16 +43,15 @@ import {registerLocaleData} from '../src/i18n/locale_data';
const u = undefined; const u = undefined;
${locales.map(locale => `${existingLocalesData[locale]}`).join('\n')} ${locales.map(locale => `${generateLocaleConstant(locale)}`).join('\n')}
let l: any; let l: any;
let locales: string[] = [];
switch (goog.LOCALE) { switch (goog.LOCALE) {
${locales.map(locale => generateCases(locale)).join('')}} ${locales.map(localeData => generateCase(localeData.locale)).join('')}}
if(l) { if (l) {
locales.forEach(locale => registerLocaleData(l, locale)); registerLocaleData(l);
} }
`; `;
} }