honeymoose
/
angular-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: update stemmer to version 2.0.0 (#41724) 

NOTE:
`stemmer` v2.0.0 switched to ES modules (see
words/stemmer@03519229c8), which means
that the only way to consume it in our CommonJS setup (for example, in
[generateKeywords][1]) is via an async `import()`.

This commit makes the `generateKeywords` processor asynchronous in order
to be able to dynamically import and use `stemmer`.

[1]: 251bec159a/aio/tools/transforms/angular-base-package/processors/generateKeywords.js

PR Close #41724
This commit is contained in:
George Kalpakas 2021-04-24 11:18:01 +03:00 committed by Jessica Janiuk
parent a938849148
commit 99f2ffc740
5 changed files with 114 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
aio/package.json
View File

@ -162,7 +162,7 @@
"rimraf": "^3.0.2",
"semver": "^7.3.5",
"shelljs": "^0.8.4",
"stemmer": "^1.0.5",
"stemmer": "^2.0.0",
"timezone-mock": "^1.1.3",
"tree-kill": "^1.1.0",
"ts-node": "^9.1.1",

3
aio/tools/transforms/.eslintrc.js
View File

@ -8,6 +8,9 @@ module.exports = {
'eslint:recommended',
'plugin:jasmine/recommended'
],
'parserOptions': {
'ecmaVersion': 2020,
},
'plugins': [
'jasmine'
],

31
aio/tools/transforms/angular-base-package/processors/generateKeywords.js vendored
View File

@ -1,7 +1,5 @@
'use strict';
const stem = require('stemmer');
/**
* @dgProcessor generateKeywordsProcessor
* @description
@ -23,7 +21,9 @@ module.exports = function generateKeywordsProcessor(log) {
},
$runAfter: ['postProcessHtml'],
$runBefore: ['writing-files'],
$process(docs) {
async $process(docs) {
const {stemmer: stem} = await import('stemmer');
const dictionary = new Map();
@ -110,15 +110,11 @@ module.exports = function generateKeywordsProcessor(log) {
data: searchData,
renderedContent: JSON.stringify(searchData)
});
}
};
};
function isString(value) {
return typeof value == 'string';
}
return docs;
function tokenize(text, ignoreWords, dictionary) {
// Helpers
function tokenize(text, ignoreWords, dictionary) {
// Split on whitespace and things that are likely to be HTML tags (this is not exhaustive but reduces the unwanted tokens that are indexed).
const rawTokens = text.split(new RegExp(
'[\\s/]+' + // whitespace
@ -155,17 +151,17 @@ function tokenize(text, ignoreWords, dictionary) {
}
return tokens;
}
}
function storeToken(token, tokens, dictionary) {
function storeToken(token, tokens, dictionary) {
token = stem(token);
if (!dictionary.has(token)) {
dictionary.set(token, dictionary.size);
}
tokens.push(dictionary.get(token));
}
}
function extractMemberTokens(doc, ignoreWords, dictionary) {
function extractMemberTokens(doc, ignoreWords, dictionary) {
if (!doc) return [];
let memberContent = [];
@ -184,4 +180,11 @@ function extractMemberTokens(doc, ignoreWords, dictionary) {
}
return memberContent;
}
}
};
};
function isString(value) {
return typeof value == 'string';
}

75
aio/tools/transforms/angular-base-package/processors/generateKeywords.spec.js
View File

@ -36,42 +36,39 @@ describe('generateKeywords processor', () => {
expect(processor.$runBefore).toEqual(['writing-files']);
});
it('should ignore internal and private exports', () => {
it('should ignore internal and private exports', async () => {
const processor = createProcessor();
const docs = [
const docs = await processor.$process([
{ docType: 'class', name: 'PublicExport' },
{ docType: 'class', name: 'PrivateExport', privateExport: true },
{ docType: 'class', name: 'InternalExport', internal: true }
];
processor.$process(docs);
]);
expect(docs[docs.length - 1].data.pages).toEqual([
jasmine.objectContaining({ title: 'PublicExport', type: 'class' })
]);
});
it('should ignore docs that are in the `docTypesToIgnore` list', () => {
it('should ignore docs that are in the `docTypesToIgnore` list', async () => {
const processor = createProcessor();
processor.docTypesToIgnore = ['interface'];
const docs = [
const docs = await processor.$process([
{ docType: 'class', name: 'Class' },
{ docType: 'interface', name: 'Interface' },
{ docType: 'content', name: 'Guide' },
];
processor.$process(docs);
]);
expect(docs[docs.length - 1].data.pages).toEqual([
jasmine.objectContaining({ title: 'Class', type: 'class' }),
jasmine.objectContaining({ title: 'Guide', type: 'content' }),
]);
});
it('should not collect keywords from properties that are in the `propertiesToIgnore` list', () => {
it('should not collect keywords from properties that are in the `propertiesToIgnore` list', async () => {
const processor = createProcessor();
processor.propertiesToIgnore = ['docType', 'ignore'];
const docs = [
const docs = await processor.$process([
{ docType: 'class', name: 'FooClass', ignore: 'ignore this content' },
{ docType: 'interface', name: 'BarInterface', capture: 'capture this content' },
];
processor.$process(docs);
]);
expect(docs[docs.length - 1].data).toEqual({
dictionary: 'fooclass barinterfac captur content',
pages: [
@ -81,17 +78,16 @@ describe('generateKeywords processor', () => {
});
});
it('should not collect keywords that look like HTML tags', () => {
it('should not collect keywords that look like HTML tags', async () => {
const processor = createProcessor();
const docs = [
const docs = await processor.$process([
{ docType: 'class', name: 'FooClass', content: `
<table id="foo">
<tr class="moo" id="bar">
<td>Content inside a table</td>
</tr>
</table>` },
];
processor.$process(docs);
]);
expect(docs[docs.length - 1].data).toEqual({
dictionary: 'class fooclass content insid tabl',
pages: [
@ -100,15 +96,14 @@ describe('generateKeywords processor', () => {
});
});
it('should compute `doc.searchTitle` from the doc properties if not already provided', () => {
it('should compute `doc.searchTitle` from the doc properties if not already provided', async () => {
const processor = createProcessor();
const docs = [
const docs = await processor.$process([
{ docType: 'class', name: 'A', searchTitle: 'searchTitle A', title: 'title A', vFile: { headings: { h1: ['vFile A'] } } },
{ docType: 'class', name: 'B', title: 'title B', vFile: { headings: { h1: ['vFile B'] } } },
{ docType: 'class', name: 'C', vFile: { title: 'vFile C', headings: { h1: ['vFile C'] } } },
{ docType: 'class', name: 'D' },
];
processor.$process(docs);
]);
expect(docs[docs.length - 1].data.pages).toEqual([
jasmine.objectContaining({ title: 'searchTitle A' }),
jasmine.objectContaining({ title: 'title B' }),
@ -117,29 +112,27 @@ describe('generateKeywords processor', () => {
]);
});
it('should use `doc.searchTitle` as the title in the search index', () => {
it('should use `doc.searchTitle` as the title in the search index', async () => {
const processor = createProcessor();
const docs = [
const docs = await processor.$process([
{ docType: 'class', name: 'PublicExport', searchTitle: 'class PublicExport' },
];
processor.$process(docs);
]);
const keywordsDoc = docs[docs.length - 1];
expect(keywordsDoc.data.pages).toEqual([
jasmine.objectContaining({ title: 'class PublicExport', type: 'class' })
]);
});
it('should add heading words to the search terms', () => {
it('should add heading words to the search terms', async () => {
const processor = createProcessor();
const docs = [
const docs = await processor.$process([
{
docType: 'class',
name: 'PublicExport',
searchTitle: 'class PublicExport',
vFile: { headings: { h2: ['Important heading', 'Secondary heading'] } }
},
];
processor.$process(docs);
]);
const keywordsDoc = docs[docs.length - 1];
expect(keywordsDoc.data).toEqual({
dictionary: 'class publicexport head secondari',
@ -149,9 +142,9 @@ describe('generateKeywords processor', () => {
});
});
it('should add member doc properties to the search terms', () => {
it('should add member doc properties to the search terms', async () => {
const processor = createProcessor();
const docs = [
const docs = await processor.$process([
{
docType: 'class',
name: 'PublicExport',
@ -171,8 +164,7 @@ describe('generateKeywords processor', () => {
{ name: 'staticPropertyB' },
],
},
];
processor.$process(docs);
]);
const keywordsDoc = docs[docs.length - 1];
expect(keywordsDoc.data).toEqual({
dictionary: 'class publicexport content ngclass instancemethoda instancepropertya instancemethodb instancepropertyb staticmethoda staticpropertya staticmethodb staticpropertyb head',
@ -184,7 +176,7 @@ describe('generateKeywords processor', () => {
});
});
it('should add inherited member doc properties to the search terms', () => {
it('should add inherited member doc properties to the search terms', async () => {
const processor = createProcessor();
const parentClass = {
docType: 'class',
@ -216,8 +208,7 @@ describe('generateKeywords processor', () => {
extendsClauses: [{ doc: parentClass }],
implementsClauses: [{ doc: parentInterface }]
};
const docs = [childClass, parentClass, parentInterface];
processor.$process(docs);
const docs = await processor.$process([childClass, parentClass, parentInterface]);
const keywordsDoc = docs[docs.length - 1];
expect(keywordsDoc.data).toEqual({
dictionary: 'class child childmember1 childmember2 parentmember1 parentmember2 parentmember3 parentclass interfac parentinterfac',
@ -238,9 +229,9 @@ describe('generateKeywords processor', () => {
});
});
it('should include both stripped and unstripped "ng" prefixed tokens', () => {
it('should include both stripped and unstripped "ng" prefixed tokens', async () => {
const processor = createProcessor();
const docs = [
const docs = await processor.$process([
{
docType: 'class',
name: 'PublicExport',
@ -248,8 +239,7 @@ describe('generateKeywords processor', () => {
vFile: { headings: { h2: ['ngModel'] } },
content: 'Some content with ngClass in it.'
},
];
processor.$process(docs);
]);
const keywordsDoc = docs[docs.length - 1];
expect(keywordsDoc.data).toEqual({
dictionary: 'class publicexport ngcontrol control content ngclass ngmodel model',
@ -262,9 +252,9 @@ describe('generateKeywords processor', () => {
});
});
it('should generate compressed encoded renderedContent property', () => {
it('should generate compressed encoded renderedContent property', async () => {
const processor = createProcessor();
const docs = [
const docs = await processor.$process([
{
docType: 'class',
name: 'SomeClass',
@ -280,8 +270,7 @@ describe('generateKeywords processor', () => {
],
deprecated: true
},
];
processor.$process(docs);
]);
const keywordsDoc = docs[docs.length - 1];
expect(JSON.parse(keywordsDoc.renderedContent)).toEqual({
dictionary: 'class someclass document api head someclass2 descript member1',

8
aio/yarn.lock
View File

@ -11955,10 +11955,10 @@ stealthy-require@^1.1.1:
resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b"
integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=
stemmer@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/stemmer/-/stemmer-1.0.5.tgz#fd89beaf8bff5d04b6643bfffcaed0fc420deec0"
integrity sha512-SLq7annzSKRDStasOJJoftCSCzBCKmBmH38jC4fDtCunAqOzpTpIm9zmaHmwNJiZ8gLe9qpVdBVbEG2DC5dE2A==
stemmer@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/stemmer/-/stemmer-2.0.0.tgz#05fcaf174c423b0fec85e660759ebd4867d811c9"
integrity sha512-0YS2oMdTZ/wAWUHMMpf7AAJ8Gm6dHXyHddJ0zCu2DIfOfIbdwqAm1bbk4+Vti6gxNIcOrnm5jAP7vYTzQDvc5A==
stream-browserify@^2.0.1:
version "2.0.2"