From 33d250ffaa45e6e6e26263bd44381cdfa8a22a38 Mon Sep 17 00:00:00 2001 From: Olivier Combe Date: Thu, 20 Jul 2017 10:54:07 +0200 Subject: [PATCH] build(common): extract i18n locale data from cldr (#18284) PR Close #18284 --- .gitignore | 1 + gulpfile.js | 2 + karma-js.conf.js | 3 +- npm-shrinkwrap.clean.json | 92 ++++- npm-shrinkwrap.json | 164 +++++++- package.json | 4 +- scripts/cldr/gen_plural_rules.js | 179 --------- tools/gulp-tasks/cldr.js | 29 ++ tools/gulp-tasks/cldr/cldr-data.js | 82 ++++ tools/gulp-tasks/cldr/cldr-urls.json | 22 ++ tools/gulp-tasks/cldr/extract.js | 539 +++++++++++++++++++++++++++ tools/gulp-tasks/format.js | 15 + 12 files changed, 923 insertions(+), 209 deletions(-) delete mode 100644 scripts/cldr/gen_plural_rules.js create mode 100644 tools/gulp-tasks/cldr.js create mode 100644 tools/gulp-tasks/cldr/cldr-data.js create mode 100644 tools/gulp-tasks/cldr/cldr-urls.json create mode 100644 tools/gulp-tasks/cldr/extract.js diff --git a/.gitignore b/.gitignore index fc1c7b2330..039f0db9c2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ bazel-* e2e_test.* node_modules bower_components +tools/gulp-tasks/cldr/cldr-data/ # Include when developing application packages. pubspec.lock diff --git a/gulpfile.js b/gulpfile.js index ff78cc396a..f3e8f3e8af 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -42,3 +42,5 @@ gulp.task('serve', loadTask('serve', 'default')); gulp.task('serve-examples', loadTask('serve', 'examples')); gulp.task('changelog', loadTask('changelog')); gulp.task('check-env', () => {/* this is a noop because the env test ran already above */}); +gulp.task('cldr:extract', loadTask('cldr', 'extract')); +gulp.task('cldr:download', loadTask('cldr', 'download')); diff --git a/karma-js.conf.js b/karma-js.conf.js index ef566dd2ff..c93da152a5 100644 --- a/karma-js.conf.js +++ b/karma-js.conf.js @@ -47,7 +47,8 @@ module.exports = function(config) { pattern: 'packages/platform-browser/test/browser/static_assets/**', included: false, watched: false, - } + }, + {pattern: 'packages/common/i18n/**', included: false, watched: false, served: true}, ], exclude: [ diff --git a/npm-shrinkwrap.clean.json b/npm-shrinkwrap.clean.json index b2d8292227..edbe15e785 100644 --- a/npm-shrinkwrap.clean.json +++ b/npm-shrinkwrap.clean.json @@ -1542,7 +1542,7 @@ } }, "cldr": { - "version": "3.5.2", + "version": "4.5.0", "dependencies": { "uglify-js": { "version": "1.3.3" @@ -1552,6 +1552,59 @@ } } }, + "cldr-data-downloader": { + "version": "0.3.2", + "dependencies": { + "adm-zip": { + "version": "0.4.4" + }, + "async": { + "version": "2.5.0" + }, + "bl": { + "version": "1.1.2" + }, + "form-data": { + "version": "1.0.1" + }, + "isarray": { + "version": "1.0.0" + }, + "lodash": { + "version": "4.17.4" + }, + "mime-db": { + "version": "1.27.0" + }, + "mime-types": { + "version": "2.1.15" + }, + "minimist": { + "version": "0.0.8" + }, + "mkdirp": { + "version": "0.5.0" + }, + "q": { + "version": "1.0.1" + }, + "qs": { + "version": "6.2.3" + }, + "readable-stream": { + "version": "2.0.6" + }, + "request": { + "version": "2.74.0" + }, + "tough-cookie": { + "version": "2.3.2" + } + } + }, + "cldrjs": { + "version": "0.5.0" + }, "cli-boxes": { "version": "1.0.0" }, @@ -1660,6 +1713,9 @@ } } }, + "config-chain": { + "version": "1.1.11" + }, "configstore": { "version": "2.1.0", "dependencies": { @@ -3717,13 +3773,10 @@ "version": "0.3.0" }, "memoizeasync": { - "version": "0.8.0", + "version": "1.0.0", "dependencies": { "lru-cache": { "version": "2.5.0" - }, - "passerror": { - "version": "0.0.2" } } }, @@ -3876,6 +3929,14 @@ "normalize-path": { "version": "2.0.1" }, + "npmconf": { + "version": "2.0.9", + "dependencies": { + "semver": { + "version": "4.3.6" + } + } + }, "npmlog": { "version": "4.0.2" }, @@ -4014,7 +4075,7 @@ "version": "2.0.0" }, "passerror": { - "version": "0.0.1" + "version": "1.1.1" }, "path-browserify": { "version": "0.0.0" @@ -4081,6 +4142,9 @@ "process-nextick-args": { "version": "1.0.6" }, + "progress": { + "version": "1.1.8" + }, "promise": { "version": "7.1.1" }, @@ -4095,6 +4159,9 @@ } } }, + "proto-list": { + "version": "1.2.4" + }, "protobufjs": { "version": "5.0.0", "dependencies": { @@ -4342,6 +4409,9 @@ "request-capture-har": { "version": "1.1.4" }, + "request-progress": { + "version": "0.3.1" + }, "requires-port": { "version": "1.0.0" }, @@ -4760,6 +4830,9 @@ "text-extensions": { "version": "1.3.3" }, + "throttleit": { + "version": "0.0.2" + }, "through": { "version": "2.3.8" }, @@ -4932,6 +5005,9 @@ "uglify-to-browserify": { "version": "1.0.2" }, + "uid-number": { + "version": "0.0.5" + }, "uid-safe": { "version": "2.0.0" }, @@ -5271,13 +5347,13 @@ "version": "8.2.2" }, "xmldom": { - "version": "0.1.19" + "version": "0.1.27" }, "xmlhttprequest-ssl": { "version": "1.5.1" }, "xpath": { - "version": "0.0.7" + "version": "0.0.24" }, "xtend": { "version": "4.0.1" diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 28c7f9a952..8f066b3fd3 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -2408,9 +2408,9 @@ } }, "cldr": { - "version": "3.5.2", - "from": "cldr@>=3.5.0 <4.0.0", - "resolved": "https://registry.npmjs.org/cldr/-/cldr-3.5.2.tgz", + "version": "4.5.0", + "from": "cldr@4.5.0", + "resolved": "https://registry.npmjs.org/cldr/-/cldr-4.5.0.tgz", "dependencies": { "uglify-js": { "version": "1.3.3", @@ -2424,6 +2424,93 @@ } } }, + "cldr-data-downloader": { + "version": "0.3.2", + "from": "cldr-data-downloader@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/cldr-data-downloader/-/cldr-data-downloader-0.3.2.tgz", + "dependencies": { + "adm-zip": { + "version": "0.4.4", + "from": "adm-zip@0.4.4", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.4.4.tgz" + }, + "async": { + "version": "2.5.0", + "from": "async@>=2.0.1 <3.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.5.0.tgz" + }, + "bl": { + "version": "1.1.2", + "from": "bl@>=1.1.2 <1.2.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.1.2.tgz" + }, + "form-data": { + "version": "1.0.1", + "from": "form-data@>=1.0.0-rc4 <1.1.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.1.tgz" + }, + "isarray": { + "version": "1.0.0", + "from": "isarray@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + }, + "lodash": { + "version": "4.17.4", + "from": "lodash@>=4.14.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz" + }, + "mime-db": { + "version": "1.27.0", + "from": "mime-db@>=1.27.0 <1.28.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.27.0.tgz" + }, + "mime-types": { + "version": "2.1.15", + "from": "mime-types@>=2.1.7 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.15.tgz" + }, + "minimist": { + "version": "0.0.8", + "from": "minimist@0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz" + }, + "mkdirp": { + "version": "0.5.0", + "from": "mkdirp@0.5.0", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.0.tgz" + }, + "q": { + "version": "1.0.1", + "from": "q@1.0.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.0.1.tgz" + }, + "qs": { + "version": "6.2.3", + "from": "qs@>=6.2.0 <6.3.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.2.3.tgz" + }, + "readable-stream": { + "version": "2.0.6", + "from": "readable-stream@>=2.0.5 <2.1.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz" + }, + "request": { + "version": "2.74.0", + "from": "request@>=2.74.0 <2.75.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.74.0.tgz" + }, + "tough-cookie": { + "version": "2.3.2", + "from": "tough-cookie@>=2.3.0 <2.4.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz" + } + } + }, + "cldrjs": { + "version": "0.5.0", + "from": "cldrjs@0.5.0", + "resolved": "https://registry.npmjs.org/cldrjs/-/cldrjs-0.5.0.tgz" + }, "cli-boxes": { "version": "1.0.0", "from": "cli-boxes@>=1.0.0 <2.0.0", @@ -2596,6 +2683,11 @@ } } }, + "config-chain": { + "version": "1.1.11", + "from": "config-chain@>=1.1.8 <1.2.0", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.11.tgz" + }, "configstore": { "version": "2.1.0", "from": "configstore@>=2.0.0 <3.0.0", @@ -5917,19 +6009,14 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" }, "memoizeasync": { - "version": "0.8.0", - "from": "memoizeasync@0.8.0", - "resolved": "https://registry.npmjs.org/memoizeasync/-/memoizeasync-0.8.0.tgz", + "version": "1.0.0", + "from": "memoizeasync@1.0.0", + "resolved": "https://registry.npmjs.org/memoizeasync/-/memoizeasync-1.0.0.tgz", "dependencies": { "lru-cache": { "version": "2.5.0", "from": "lru-cache@2.5.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.0.tgz" - }, - "passerror": { - "version": "0.0.2", - "from": "passerror@0.0.2", - "resolved": "https://registry.npmjs.org/passerror/-/passerror-0.0.2.tgz" } } }, @@ -6176,6 +6263,18 @@ "from": "normalize-path@>=2.0.1 <3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.0.1.tgz" }, + "npmconf": { + "version": "2.0.9", + "from": "npmconf@2.0.9", + "resolved": "https://registry.npmjs.org/npmconf/-/npmconf-2.0.9.tgz", + "dependencies": { + "semver": { + "version": "4.3.6", + "from": "semver@>=2.0.0 <3.0.0||>=3.0.0 <4.0.0||>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz" + } + } + }, "npmlog": { "version": "4.0.2", "from": "npmlog@>=0.0.0 <1.0.0||>=1.0.0 <2.0.0||>=2.0.0 <3.0.0||>=3.0.0 <4.0.0||>=4.0.0 <5.0.0", @@ -6400,9 +6499,9 @@ "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-2.0.0.tgz" }, "passerror": { - "version": "0.0.1", - "from": "passerror@0.0.1", - "resolved": "https://registry.npmjs.org/passerror/-/passerror-0.0.1.tgz" + "version": "1.1.1", + "from": "passerror@1.1.1", + "resolved": "https://registry.npmjs.org/passerror/-/passerror-1.1.1.tgz" }, "path-browserify": { "version": "0.0.0", @@ -6511,6 +6610,11 @@ "from": "process-nextick-args@>=1.0.6 <1.1.0", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.6.tgz" }, + "progress": { + "version": "1.1.8", + "from": "progress@1.1.8", + "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz" + }, "promise": { "version": "7.1.1", "from": "promise@>=7.0.3 <8.0.0", @@ -6533,6 +6637,11 @@ } } }, + "proto-list": { + "version": "1.2.4", + "from": "proto-list@>=1.2.1 <1.3.0", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz" + }, "protobufjs": { "version": "5.0.0", "from": "protobufjs@5.0.0", @@ -6934,6 +7043,11 @@ "from": "request-capture-har@>=1.1.4 <2.0.0", "resolved": "https://registry.npmjs.org/request-capture-har/-/request-capture-har-1.1.4.tgz" }, + "request-progress": { + "version": "0.3.1", + "from": "request-progress@0.3.1", + "resolved": "https://registry.npmjs.org/request-progress/-/request-progress-0.3.1.tgz" + }, "requires-port": { "version": "1.0.0", "from": "requires-port@>=1.0.0 <2.0.0", @@ -7604,6 +7718,11 @@ "from": "text-extensions@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-1.3.3.tgz" }, + "throttleit": { + "version": "0.0.2", + "from": "throttleit@>=0.0.2 <0.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-0.0.2.tgz" + }, "through": { "version": "2.3.8", "from": "through@>=2.2.7 <3.0.0", @@ -7880,6 +7999,11 @@ "from": "uglify-to-browserify@>=1.0.0 <1.1.0", "resolved": "https://registry.npmjs.org/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz" }, + "uid-number": { + "version": "0.0.5", + "from": "uid-number@0.0.5", + "resolved": "https://registry.npmjs.org/uid-number/-/uid-number-0.0.5.tgz" + }, "uid-safe": { "version": "2.0.0", "from": "uid-safe@>=2.0.0 <2.1.0", @@ -8427,9 +8551,9 @@ "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-8.2.2.tgz" }, "xmldom": { - "version": "0.1.19", - "from": "xmldom@0.1.19", - "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.1.19.tgz" + "version": "0.1.27", + "from": "xmldom@0.1.27", + "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.1.27.tgz" }, "xmlhttprequest-ssl": { "version": "1.5.1", @@ -8437,9 +8561,9 @@ "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.1.tgz" }, "xpath": { - "version": "0.0.7", - "from": "xpath@0.0.7", - "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.7.tgz" + "version": "0.0.24", + "from": "xpath@0.0.24", + "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.24.tgz" }, "xtend": { "version": "4.0.1", diff --git a/package.json b/package.json index be1d406933..eaf31926d8 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,9 @@ "canonical-path": "0.0.2", "chokidar": "^1.1.0", "clang-format": "^1.0.32", - "cldr": "^3.5.2", + "cldr": "^4.5.0", + "cldr-data-downloader": "^0.3.2", + "cldrjs": "^0.5.0", "conventional-changelog": "^1.1.0", "cors": "^2.7.1", "dgeni": "^0.4.2", diff --git a/scripts/cldr/gen_plural_rules.js b/scripts/cldr/gen_plural_rules.js deleted file mode 100644 index 0000a82949..0000000000 --- a/scripts/cldr/gen_plural_rules.js +++ /dev/null @@ -1,179 +0,0 @@ -/** - * @license - * Copyright Google Inc. All Rights Reserved. - * - * Use of this source code is governed by an MIT-style license that can be - * found in the LICENSE file at https://angular.io/license - */ - -const cldr = require('cldr'); -// locale list -const locales = cldr.localeIds; -const langToRule = {}; -const ruleToLang = {}; -const variants = []; -const localeToVariant = {}; -const DEFAULT_RULE = `function anonymous(n\n/**/) {\nreturn"other"\n}`; -const EMPTY_RULE = `function anonymous(n\n/**/) {\n\n}`; - -locales.forEach(locale => { - const rule = normalizeRule(cldr.extractPluralRuleFunction(locale).toString()); - const lang = getVariantLang(locale, rule); - - if (!lang || !rule) { - return; - } - - if (!ruleToLang[rule]) { - ruleToLang[rule] = []; - } else if (ruleToLang[rule].indexOf(lang) > -1) { - return; - } - - ruleToLang[rule].push(lang); -}); - -let nextVariantCode = 'a'.charCodeAt(0); - -variants.forEach(locale => { - const rule = normalizeRule(cldr.extractPluralRuleFunction(locale).toString()); - if (!rule) { - return; - } - - let mapTo = null; - - if (ruleToLang[rule]) { - mapTo = ruleToLang[rule][0]; - localeToVariant[locale] = mapTo; - return; - } - - if (!mapTo) { - mapTo = '_' + String.fromCharCode(nextVariantCode++); - - langToRule[mapTo] = rule; - ruleToLang[rule] = [mapTo]; - localeToVariant[locale] = mapTo; - } -}); - -console.log(generateCode()); - -function generateCode() { - checkMapping(); - - return ` -// This is generated code DO NOT MODIFY -// see angular/script/cldr/gen_plural_rules.js - -/** @experimental */ -export enum Plural { - Zero, - One, - Two, - Few, - Many, - Other, -} -` + generateVars() + - generateRules() + ` -}`; -} - -function generateRules() { - const codeParts = [` -const lang = locale.split('-')[0].toLowerCase(); - -switch (lang) {`]; - - Object.keys(ruleToLang).forEach(rule => { - const langs = ruleToLang[rule]; - codeParts.push(...langs.map(l => ` case '${l}': `)); - codeParts.push(` ${rule}`); - }); - - codeParts.push(` // When there is no specification, the default is always other - // see http://cldr.unicode.org/index/cldr-spec/plural-rules - // "other (required—general plural form — also used if the language only has a single form)" - default: - return Plural.Other; -}`); - - return codeParts.join('\n'); -} - -function generateVars(){ - return ` -/** - * Returns the plural case based on the locale - * - * @experimental - */ -export function getPluralCase(locale: string, nLike: number | string): Plural { -// TODO(vicb): lazy compute -if (typeof nLike === 'string') { - nLike = parseInt(nLike, 10); -} -const n: number = nLike as number; -const nDecimal = n.toString().replace(/^[^.]*\\.?/, ''); -const i = Math.floor(Math.abs(n)); -const v = nDecimal.length; -const f = parseInt(nDecimal, 10); -const t = parseInt(n.toString().replace(/^[^.]*\\.?|0+$/g,''), 10) || 0; -`; -} - -function checkMapping() { - if (localeToVariant.length) { - console.log(`Mapping required:`); - console.log(localeToVariant); - throw new Error('not implemented'); - } -} - -/** - * If the language rule do not match an existing language rule, flag it as variant and handle it at the end - */ -function getVariantLang(locale, rule) { - let lang = locale.split('_')[0]; - - if (!langToRule[lang]) { - langToRule[lang] = rule; - return lang; - } - - if (langToRule[lang] === rule) { - return lang; - } - - variants.push(locale); - return null; -} - -function normalizeRule(fn) { - if (fn === DEFAULT_RULE || fn === EMPTY_RULE) return; - - return fn - .replace(toRegExp('function anonymous(n\n/**/) {\n'), '') - .replace(toRegExp('var'), 'let') - .replace(toRegExp('"zero"'), ' Plural.Zero') - .replace(toRegExp('"one"'), ' Plural.One') - .replace(toRegExp('"two"'), ' Plural.Two') - .replace(toRegExp('"few"'), ' Plural.Few') - .replace(toRegExp('"many"'), ' Plural.Many') - .replace(toRegExp('"other"'), ' Plural.Other') - .replace(toRegExp('\n}'), '') - .replace(toRegExp('let'), '') - .replace(toRegExp('if(typeof n==="string")n=parseInt(n,10);'), '') - .replace(toRegExp('i=Math.floor(Math.abs(n))'), '') - .replace(/v=n.toString.*?.length/g, '') - .replace(/f=parseInt.*?\|\|0/g, '') - .replace(/t=parseInt.*?\|\|0/g, '') - .replace(/^[ ,;]*/, '') - + ';'; -} - -function toRegExp(s) { - return new RegExp(s.replace(/([.*+?^=!:${}()|[\]\/\\])/g, '\\$1'), 'g'); -} diff --git a/tools/gulp-tasks/cldr.js b/tools/gulp-tasks/cldr.js new file mode 100644 index 0000000000..e6785161e9 --- /dev/null +++ b/tools/gulp-tasks/cldr.js @@ -0,0 +1,29 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +const path = require('path'); +const fs = require('fs'); + +module.exports = { + extract: gulp => done => { + if (!fs.existsSync(path.join(__dirname, 'cldr/cldr-data'))) { + throw new Error(`You must run "gulp cldr:download" before you can extract the data`); + } + const extract = require('./cldr/extract'); + return extract(gulp, done); + }, + + download: gulp => done => { + const cldrDownloader = require('cldr-data-downloader'); + const cldrDataFolder = path.join(__dirname, 'cldr/cldr-data'); + if (!fs.existsSync(cldrDataFolder)) { + fs.mkdirSync(cldrDataFolder); + } + cldrDownloader(path.join(__dirname, 'cldr/cldr-urls.json'), cldrDataFolder, done); + } +}; diff --git a/tools/gulp-tasks/cldr/cldr-data.js b/tools/gulp-tasks/cldr/cldr-data.js new file mode 100644 index 0000000000..ab502d36ef --- /dev/null +++ b/tools/gulp-tasks/cldr/cldr-data.js @@ -0,0 +1,82 @@ +/** + * Npm module for Unicode CLDR JSON data + * + * @license + * Copyright 2013 Rafael Xavier de Souza + * Released under the MIT license + * https://github.com/rxaviers/cldr-data-npm/blob/master/LICENSE-MIT + */ + +'use strict'; + +const JSON_EXTENSION = /^(.*)\.json$/; + +const assert = require('assert'); +const _fs = require('fs'); +const _path = require('path'); + +function argsToArray(arg) { + return [].slice.call(arg, 0); +} + +function jsonFiles(dirName) { + const fileList = _fs.readdirSync(_path.join(__dirname, 'cldr-data', dirName)); + + return fileList.reduce(function(sum, file) { + if (JSON_EXTENSION.test(file)) { + return sum.concat(file.match(JSON_EXTENSION)[1]); + } + }, []); +} + +function cldrData(path /*, ...*/) { + assert( + typeof path === 'string', 'must include path (e.g., ' + + '"main/en/numbers" or "supplemental/likelySubtags")'); + + if (arguments.length > 1) { + return argsToArray(arguments).reduce(function(sum, path) { + sum.push(cldrData(path)); + return sum; + }, []); + } + return require('./cldr-data/' + path); +} + +function mainPathsFor(locales) { + return locales.reduce(function(sum, locale) { + const mainFiles = jsonFiles(_path.join('main', locale)); + mainFiles.forEach(function(mainFile) { sum.push(_path.join('main', locale, mainFile)); }); + return sum; + }, []); +} + +function supplementalPaths() { + const supplementalFiles = jsonFiles('supplemental'); + + return supplementalFiles.map(function(supplementalFile) { + return _path.join('supplemental', supplementalFile); + }); +} + +Object.defineProperty( + cldrData, 'availableLocales', + {get: function() { return cldrData('availableLocales').availableLocales; }}); + +cldrData.all = function() { + const paths = supplementalPaths().concat(mainPathsFor(this.availableLocales)); + return cldrData.apply({}, paths); +}; + +cldrData.entireMainFor = function(locale /*, ...*/) { + assert( + typeof locale === 'string', 'must include locale (e.g., ' + + '"en")'); + return cldrData.apply({}, mainPathsFor(argsToArray(arguments))); +}; + +cldrData.entireSupplemental = function() { + return cldrData.apply({}, supplementalPaths()); +}; + +module.exports = cldrData; diff --git a/tools/gulp-tasks/cldr/cldr-urls.json b/tools/gulp-tasks/cldr/cldr-urls.json new file mode 100644 index 0000000000..c5f6500ebc --- /dev/null +++ b/tools/gulp-tasks/cldr/cldr-urls.json @@ -0,0 +1,22 @@ +{ + "core": [ + "https://github.com/unicode-cldr/cldr-core/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-segments-modern/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-dates-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-buddhist-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-chinese-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-coptic-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-dangi-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-ethiopic-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-hebrew-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-indian-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-islamic-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-japanese-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-persian-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-cal-roc-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-localenames-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-misc-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-numbers-full/archive/31.0.1.zip", + "https://github.com/unicode-cldr/cldr-units-full/archive/31.0.1.zip" + ] +} diff --git a/tools/gulp-tasks/cldr/extract.js b/tools/gulp-tasks/cldr/extract.js new file mode 100644 index 0000000000..dcc602a0ae --- /dev/null +++ b/tools/gulp-tasks/cldr/extract.js @@ -0,0 +1,539 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +const fs = require('fs'); +const path = require('path'); +const util = require('util'); +// used to extract plural rules +const cldr = require('cldr'); +// used to extract all other cldr data +const cldrJs = require('cldrjs'); + +const PACKAGE_FOLDER = 'packages/common'; +const I18N_FOLDER = `${PACKAGE_FOLDER}/src/i18n`; +const I18N_DATA_FOLDER = `${PACKAGE_FOLDER}/i18n_data`; +const I18N_DATA_EXTRA_FOLDER = `${I18N_DATA_FOLDER}/extra`; +const RELATIVE_I18N_FOLDER = path.resolve(__dirname, `../../../${I18N_FOLDER}`); +const RELATIVE_I18N_DATA_FOLDER = path.resolve(__dirname, `../../../${I18N_DATA_FOLDER}`); +const RELATIVE_I18N_DATA_EXTRA_FOLDER = path.resolve(__dirname, `../../../${I18N_DATA_EXTRA_FOLDER}`); +const DEFAULT_RULE = `function anonymous(n\n/**/) {\nreturn"other"\n}`; +const EMPTY_RULE = `function anonymous(n\n/**/) {\n\n}`; +const WEEK_DAYS = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat']; +const HEADER = `/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +// THIS CODE IS GENERATED - DO NOT MODIFY +// See angular/tools/gulp-tasks/cldr/extract.js +`; + +module.exports = (gulp, done) => { + const cldrData = require('./cldr-data'); + const LOCALES = cldrData.availableLocales; + + console.log(`Loading CLDR data...`); + cldrJs.load(cldrData.all()); + + console.log(`Writing locale files`); + if (!fs.existsSync(RELATIVE_I18N_FOLDER)) { + fs.mkdirSync(RELATIVE_I18N_FOLDER); + } + if (!fs.existsSync(RELATIVE_I18N_DATA_FOLDER)) { + fs.mkdirSync(RELATIVE_I18N_DATA_FOLDER); + } + if (!fs.existsSync(RELATIVE_I18N_DATA_EXTRA_FOLDER)) { + fs.mkdirSync(RELATIVE_I18N_DATA_EXTRA_FOLDER); + } + LOCALES.forEach((locale, index) => { + const localeData = new cldrJs(locale); + + console.log(`${index + 1}/${LOCALES.length}`); + console.log(`\t${I18N_DATA_FOLDER}/locale_${locale}.ts`); + fs.writeFileSync(`${RELATIVE_I18N_DATA_FOLDER}/locale_${locale}.ts`, generateLocale(locale, localeData, '@angular/common')); + console.log(`\t${I18N_DATA_EXTRA_FOLDER}/locale_${locale}.ts`); + fs.writeFileSync(`${RELATIVE_I18N_DATA_EXTRA_FOLDER}/locale_${locale}.ts`, generateLocaleExtra(locale, localeData)); + }); + console.log(`${LOCALES.length} locale files generated.`); + + // additional "en" file that will be included in common + console.log(`Writing file ${I18N_FOLDER}/locale_en.ts`); + fs.writeFileSync(`${RELATIVE_I18N_FOLDER}/locale_en.ts`, generateLocale('en', new cldrJs('en'), './locale_data')); + + console.log(`Writing file ${I18N_FOLDER}/available_locales.ts`); + fs.writeFileSync(`${RELATIVE_I18N_FOLDER}/available_locales.ts`, generateAvailableLocales(LOCALES)); + + console.log(`Writing file ${I18N_FOLDER}/currencies.ts`); + fs.writeFileSync(`${RELATIVE_I18N_FOLDER}/currencies.ts`, generateCurrencies()); + + console.log(`All i18n cldr files have been generated, formatting files..."`); + const format = require('gulp-clang-format'); + const clangFormat = require('clang-format'); + return gulp + .src([ + `${I18N_DATA_FOLDER}/**/*.ts`, + `${I18N_FOLDER}/available_locales.ts`, + `${I18N_FOLDER}/currencies.ts`, + `${I18N_FOLDER}/locale_en.ts` + ], {base: '.'}) + .pipe(format.format('file', clangFormat)) + .pipe(gulp.dest('.')); +}; + +/** + * Generate file that contains basic locale data + */ +function generateLocale(locale, localeData, ngLocalePath) { + // [ localeId, dateTime, number, currency, pluralCase ] + let data = stringify([ + locale, + ...getDateTimeTranslations(localeData), + ...getDateTimeSettings(localeData), + ...getNumberSettings(localeData), + ...getCurrencySettings(locale, localeData) + ]) + // We remove "undefined" added by spreading arrays when there is no value + .replace(/undefined/g, ''); + + // adding plural function after, because we don't want it as a string + data = data.substring(0, data.lastIndexOf(']')) + `, ${getPluralFunction(locale)}]`; + + return `${HEADER} +import {Plural} from '${ngLocalePath}'; + +export default ${data}; +`; +} + +/** + * Generate a file that contains extra data (for now: day period rules, and extended day period data) + */ +function generateLocaleExtra(locale, localeData) { + const dayPeriods = getDayPeriodsNoAmPm(localeData); + const dayPeriodRules = getDayPeriodRules(localeData); + + let dayPeriodsSupplemental = []; + + if (Object.keys(dayPeriods.format.narrow).length) { + const keys = Object.keys(dayPeriods.format.narrow); + + if (keys.length !== Object.keys(dayPeriodRules).length) { + throw new Error(`Error: locale ${locale} has not the correct number of day period rules`); + } + + const dayPeriodsFormat = removeDuplicates([ + objectValues(dayPeriods.format.narrow), + objectValues(dayPeriods.format.abbreviated), + objectValues(dayPeriods.format.wide) + ]); + + const dayPeriodsStandalone = removeDuplicates([ + objectValues(dayPeriods['stand-alone'].narrow), + objectValues(dayPeriods['stand-alone'].abbreviated), + objectValues(dayPeriods['stand-alone'].wide) + ]); + + const rules = keys.map(key => dayPeriodRules[key]); + + dayPeriodsSupplemental = [...removeDuplicates([dayPeriodsFormat, dayPeriodsStandalone]), rules]; + } + + return `${HEADER} +export default ${stringify(dayPeriodsSupplemental).replace(/undefined/g, '')}; +`; +} + +/** + * Generate a file that contains the complete list of locales + */ +function generateAvailableLocales(LOCALES) { + return `${HEADER} +/** @experimental */ +export const AVAILABLE_LOCALES = ${stringify(LOCALES)}; +`; +} + +/** + * Generate a file that contains the list of currencies and their symbols + */ +function generateCurrencies() { + const currenciesData = new cldrJs('en').main('numbers/currencies'); + const currencies = []; + Object.keys(currenciesData).forEach(key => { + let symbolsArray = []; + const symbol = currenciesData[key].symbol; + const symbolNarrow = currenciesData[key]['symbol-alt-narrow']; + if (symbol && symbol !== key) { + symbolsArray.push(symbol); + } + if (symbolNarrow && symbolNarrow !== symbol) { + if (symbolsArray.length > 0) { + symbolsArray.push(symbolNarrow); + } else { + symbolsArray = [, symbolNarrow]; + } + } + if (symbolsArray.length > 0) { + currencies.push(` '${key}': ${stringify(symbolsArray)},\n`); + } + }); + + return `${HEADER} +/** @experimental */ +export const CURRENCIES: {[code: string]: (string | undefined)[]} = { +${currencies.join('')}}; +`; +} + +/** + * Returns data for the chosen day periods + * @returns {format: {narrow / abbreviated / wide: [...]}, stand-alone: {narrow / abbreviated / wide: [...]}} + */ +function getDayPeriods(localeData, dayPeriodsList) { + const dayPeriods = localeData.main(`dates/calendars/gregorian/dayPeriods`); + const result = {}; + // cleaning up unused keys + Object.keys(dayPeriods).forEach(key1 => { // format / stand-alone + result[key1] = {}; + Object.keys(dayPeriods[key1]).forEach(key2 => { // narrow / abbreviated / wide + result[key1][key2] = {}; + Object.keys(dayPeriods[key1][key2]).forEach(key3 => { + if (dayPeriodsList.indexOf(key3) !== -1) { + result[key1][key2][key3] = dayPeriods[key1][key2][key3]; + } + }); + }); + }); + + return result; +} + +/** + * Returns the basic day periods (am/pm) + */ +function getDayPeriodsAmPm(localeData) { + return getDayPeriods(localeData, ['am', 'pm']); +} + +/** + * Returns the extra day periods (without am/pm) + */ +function getDayPeriodsNoAmPm(localeData) { + return getDayPeriods(localeData, ['noon', 'midnight', 'morning1', 'morning2', 'afternoon1', + 'afternoon2', 'evening1', 'evening2', 'night1', 'night2']); +} + +/** + * Returns date-related translations for a locale + * @returns [ dayPeriodsFormat, dayPeriodsStandalone, daysFormat, dayStandalone, monthsFormat, monthsStandalone, eras ] + * each value: [ narrow, abbreviated, wide, short? ] + */ +function getDateTimeTranslations(localeData) { + const dayNames = localeData.main(`dates/calendars/gregorian/days`); + const monthNames = localeData.main(`dates/calendars/gregorian/months`); + const erasNames = localeData.main(`dates/calendars/gregorian/eras`); + const dayPeriods = getDayPeriodsAmPm(localeData); + + const dayPeriodsFormat = removeDuplicates([ + objectValues(dayPeriods.format.narrow), + objectValues(dayPeriods.format.abbreviated), + objectValues(dayPeriods.format.wide) + ]); + + const dayPeriodsStandalone = removeDuplicates([ + objectValues(dayPeriods['stand-alone'].narrow), + objectValues(dayPeriods['stand-alone'].abbreviated), + objectValues(dayPeriods['stand-alone'].wide) + ]); + + const daysFormat = removeDuplicates([ + objectValues(dayNames.format.narrow), + objectValues(dayNames.format.abbreviated), + objectValues(dayNames.format.wide), + objectValues(dayNames.format.short) + ]); + + const daysStandalone = removeDuplicates([ + objectValues(dayNames['stand-alone'].narrow), + objectValues(dayNames['stand-alone'].abbreviated), + objectValues(dayNames['stand-alone'].wide), + objectValues(dayNames['stand-alone'].short) + ]); + + const monthsFormat = removeDuplicates([ + objectValues(monthNames.format.narrow), + objectValues(monthNames.format.abbreviated), + objectValues(monthNames.format.wide) + ]); + + const monthsStandalone = removeDuplicates([ + objectValues(monthNames['stand-alone'].narrow), + objectValues(monthNames['stand-alone'].abbreviated), + objectValues(monthNames['stand-alone'].wide) + ]); + + const eras = removeDuplicates([ + [erasNames.eraNarrow['0'], erasNames.eraNarrow['1']], + [erasNames.eraAbbr['0'], erasNames.eraAbbr['1']], + [erasNames.eraNames['0'], erasNames.eraNames['1']] + ]); + + const dateTimeTranslations = [ + ...removeDuplicates([dayPeriodsFormat, dayPeriodsStandalone]), + ...removeDuplicates([daysFormat, daysStandalone]), + ...removeDuplicates([monthsFormat, monthsStandalone]), + eras + ]; + + return dateTimeTranslations; +} + +/** + * Returns date, time and dateTime formats for a locale + * @returns [dateFormats, timeFormats, dateTimeFormats] + * each format: [ short, medium, long, full ] + */ +function getDateTimeFormats(localeData) { + function getFormats(data) { + return removeDuplicates([ + data.short._value || data.short, + data.medium._value || data.medium, + data.long._value || data.long, + data.full._value || data.full + ]) + } + + const dateFormats = localeData.main('dates/calendars/gregorian/dateFormats'); + const timeFormats = localeData.main('dates/calendars/gregorian/timeFormats'); + const dateTimeFormats = localeData.main('dates/calendars/gregorian/dateTimeFormats'); + + return [ + getFormats(dateFormats), + getFormats(timeFormats), + getFormats(dateTimeFormats) + ]; +} + +/** + * Returns day period rules for a locale + * @returns string[] + */ +function getDayPeriodRules(localeData) { + const dayPeriodRules = localeData.get(`supplemental/dayPeriodRuleSet/${localeData.attributes.language}`); + const rules = {}; + if (dayPeriodRules) { + Object.keys(dayPeriodRules).forEach(key => { + if (dayPeriodRules[key]._at) { + rules[key] = dayPeriodRules[key]._at; + } else { + rules[key] = [dayPeriodRules[key]._from, dayPeriodRules[key]._before] + } + }) + } + + return rules; +} + +/** + * Returns the first day of the week, based on US week days + * @returns number + */ +function getFirstDayOfWeek(localeData) { + return WEEK_DAYS.indexOf(localeData.supplemental.weekData.firstDay()); +} + +/** + * Returns week-end range for a locale, based on US week days + * @returns [number, number] + */ +function getWeekendRange(localeData) { + const startDay = + localeData.get(`supplemental/weekData/weekendStart/${localeData.attributes.territory}`) || + localeData.get('supplemental/weekData/weekendStart/001'); + const endDay = + localeData.get(`supplemental/weekData/weekendEnd/${localeData.attributes.territory}`) || + localeData.get('supplemental/weekData/weekendEnd/001'); + return [WEEK_DAYS.indexOf(startDay), WEEK_DAYS.indexOf(endDay)]; +} + +/** + * Returns dateTime data for a locale + * @returns [ firstDayOfWeek, weekendRange, formats ] + */ +function getDateTimeSettings(localeData) { + return [getFirstDayOfWeek(localeData), getWeekendRange(localeData), ...getDateTimeFormats(localeData)]; +} + +/** + * Returns the number symbols and formats for a locale + * @returns [ symbols, formats ] + * symbols: [ decimal, group, list, percentSign, plusSign, minusSign, exponential, superscriptingExponent, perMille, infinity, nan, timeSeparator, currencyDecimal?, currencyGroup? ] + * formats: [ currency, decimal, percent, scientific ] + */ +function getNumberSettings(localeData) { + const decimalFormat = localeData.main('numbers/decimalFormats-numberSystem-latn/standard'); + const percentFormat = localeData.main('numbers/percentFormats-numberSystem-latn/standard'); + const scientificFormat = localeData.main('numbers/scientificFormats-numberSystem-latn/standard'); + const currencyFormat = localeData.main('numbers/currencyFormats-numberSystem-latn/standard'); + const symbols = localeData.main('numbers/symbols-numberSystem-latn'); + const symbolValues = [ + symbols.decimal, + symbols.group, + symbols.list, + symbols.percentSign, + symbols.plusSign, + symbols.minusSign, + symbols.exponential, + symbols.superscriptingExponent, + symbols.perMille, + symbols.infinity, + symbols.nan, + symbols.timeSeparator, + ]; + + if (symbols.currencyDecimal) { + symbolValues.push(symbols.currencyDecimal); + } + + if (symbols.currencyGroup) { + symbolValues.push(symbols.currencyGroup); + } + + return [ + symbolValues, + [decimalFormat, percentFormat, currencyFormat, scientificFormat] + ]; +} + +/** + * Returns the currency symbol and name for a locale + * @returns [ symbol, name ] + */ +function getCurrencySettings(locale, localeData) { + const currencyInfo = localeData.main(`numbers/currencies`); + let currentCurrency = ''; + + // find the currency currently used in this country + const currencies = + localeData.get(`supplemental/currencyData/region/${localeData.attributes.territory}`) || + localeData.get(`supplemental/currencyData/region/${localeData.attributes.language.toUpperCase()}`); + + if (currencies) { + currencies.some(currency => { + const keys = Object.keys(currency); + return keys.some(key => { + if (currency[key]._from && !currency[key]._to) { + return currentCurrency = key; + } + }) + }); + + if (!currentCurrency) { + throw new Error(`Unable to find currency for locale "${locale}"`); + } + } + + let currencySettings = [,]; + + if (currentCurrency) { + currencySettings = [currencyInfo[currentCurrency].symbol, currencyInfo[currentCurrency].displayName]; + } + + return currencySettings; +} + +/** + * Transforms a string into a regexp + */ +function toRegExp(s) { + return new RegExp(s.replace(/([.*+?^=!:${}()|[\]\/\\])/g, '\\$1'), 'g'); +} + +/** + * Returns the plural function for a locale + * todo(ocombe): replace "cldr" extractPluralRuleFunction with our own extraction using "CldrJS" + * because the 2 libs can become out of sync if they use different versions of the cldr database + */ +function getPluralFunction(locale) { + let fn = cldr.extractPluralRuleFunction(locale).toString(); + if (fn === EMPTY_RULE) { + fn = DEFAULT_RULE; + } + + return fn + .replace( + toRegExp('function anonymous(n\n/**/) {\n'), + 'function(n: number): Plural {\n ') + .replace(toRegExp('var'), 'let') + .replace(toRegExp('if(typeof n==="string")n=parseInt(n,10);'), '') + .replace(toRegExp('"zero"'), ' Plural.Zero') + .replace(toRegExp('"one"'), ' Plural.One') + .replace(toRegExp('"two"'), ' Plural.Two') + .replace(toRegExp('"few"'), ' Plural.Few') + .replace(toRegExp('"many"'), ' Plural.Many') + .replace(toRegExp('"other"'), ' Plural.Other') + .replace(toRegExp('\n}'), ';\n}'); + return normalizePluralRule(); +} + +/** + * Return an array of values from an object + */ +function objectValues(obj) { + return Object.keys(obj).map(key => obj[key]); +} + +/** + * Like JSON.stringify, but without double quotes around keys, and already formatted for readability + */ +function stringify(obj) { + return util.inspect(obj, {depth: null, maxArrayLength: null}) +} + +/** + * Transform a string to camelCase + */ +function toCamelCase(str) { + return str.replace(/-+([a-z0-9A-Z])/g, (...m) => m[1].toUpperCase()); +} + +/** + * To create smaller locale files, we remove duplicated data. + * To be make this work we need to store similar data in arrays, if some value in an array + * is undefined, we can take the previous defined value instead, because it means that it has + * been deduplicated. + * e.g.: [x, y, undefined, z, undefined, undefined] + * The first undefined is equivalent to y, the second and third are equivalent to z + * Note that the first value in an array is always defined. + * + * Also since we need to know which data is assumed similar, it is important that we store those + * similar data in arrays to mark the delimitation between values that have different meanings + * (e.g. months and days). + * + * For further size improvements, "undefined" values will be replaced by empty values in the arrays + * as the last step of the file generation (in generateLocale and generateLocaleExtra). + * e.g.: [x, y, undefined, z, undefined, undefined] will be [x, y, , z, , ] + * This is possible because empty values are considered undefined in arrays. + */ +function removeDuplicates(data) { + const dedup = [data[0]]; + for(let i = 1; i < data.length; i++) { + if (stringify(data[i]) !== stringify(data[i - 1])) { + dedup.push(data[i]); + } else { + dedup.push(undefined); + } + } + return dedup; +} + +module.exports.I18N_FOLDER = I18N_FOLDER; +module.exports.I18N_DATA_FOLDER = I18N_DATA_FOLDER; diff --git a/tools/gulp-tasks/format.js b/tools/gulp-tasks/format.js index 02f27466ce..51ec44c1e6 100644 --- a/tools/gulp-tasks/format.js +++ b/tools/gulp-tasks/format.js @@ -1,3 +1,13 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +const {I18N_FOLDER, I18N_DATA_FOLDER} = require('./cldr/extract'); + // clang-format entry points const srcsToFmt = [ 'packages/**/*.{js,ts}', @@ -8,6 +18,11 @@ const srcsToFmt = [ '!tools/public_api_guard/**/*.d.ts', './*.{js,ts}', '!shims_for_IE.js', + `!${I18N_DATA_FOLDER}/**/*.{js,ts}`, + `!${I18N_FOLDER}/available_locales.ts`, + `!${I18N_FOLDER}/currencies.ts`, + `!${I18N_FOLDER}/locale_en.ts`, + '!tools/gulp-tasks/cldr/extract.js', ]; module.exports = {