build(docs-infra): use case-insensitive encoding for content files (#42414)
To avoid having content files that have the same file path on case-insensitive file-systems, we now encode the paths to remove uppercase characters. PR Close #42414
This commit is contained in:
Pete Bacon Darwin
Dylan Hunn
parent
07c1ddc487
commit
b0592c1be6
|
|
@ -67,29 +67,17 @@ describe('DocumentService', () => {
|
|||
expect(latestDocument).toEqual(doc1);
|
||||
});
|
||||
|
||||
// HACK: PREPARE FOR CHANGING TO CASE-INSENSITIVE URLS
|
||||
it('should attempt disambiguated document paths if the document is not found on the server', () => {
|
||||
let currentDocument: DocumentContents|undefined;
|
||||
const notFoundDoc = { id: FILE_NOT_FOUND_ID, contents: '<h1>Page Not Found</h1>' };
|
||||
const { docService, logger } = getServices('missing/Doc-1');
|
||||
docService.currentDocument.subscribe(doc => currentDocument = doc);
|
||||
|
||||
// Initial request return 404.
|
||||
httpMock.expectOne({url: 'generated/docs/missing/Doc-1.json'}).flush(null, {status: 404, statusText: 'NOT FOUND'});
|
||||
httpMock.expectOne({url: 'generated/docs/missing/d_oc-1.json'}).flush(null, {status: 404, statusText: 'NOT FOUND'});
|
||||
httpMock.expectOne({url: 'generated/docs/missing/d_oc.json'}).flush(null, {status: 404, statusText: 'NOT FOUND'});
|
||||
expect(logger.output.error).toEqual([
|
||||
[jasmine.any(Error)]
|
||||
]);
|
||||
expect(logger.output.error[0][0].message).toEqual(`Document file not found at 'missing/Doc-1'`);
|
||||
|
||||
// Subsequent request for not-found document.
|
||||
logger.output.error = [];
|
||||
httpMock.expectOne(CONTENT_URL_PREFIX + 'file-not-found.json').flush(notFoundDoc);
|
||||
expect(logger.output.error).toEqual([]); // does not report repeated errors
|
||||
expect(currentDocument).toEqual(notFoundDoc);
|
||||
it('should encode the request path to be case-insensitive', () => {
|
||||
const { docService, locationService } = getServices('initial/Doc');
|
||||
docService.currentDocument.subscribe();
|
||||
httpMock.expectOne(CONTENT_URL_PREFIX + 'initial/d_oc.json').flush({});
|
||||
locationService.go('NEW/Doc');
|
||||
httpMock.expectOne(CONTENT_URL_PREFIX + 'n_e_w_/d_oc.json').flush({});
|
||||
locationService.go('doc_with_underscores');
|
||||
httpMock.expectOne(CONTENT_URL_PREFIX + 'doc__with__underscores.json').flush({});
|
||||
locationService.go('DOC_WITH_UNDERSCORES');
|
||||
httpMock.expectOne(CONTENT_URL_PREFIX + 'd_o_c___w_i_t_h___u_n_d_e_r_s_c_o_r_e_s_.json').flush({});
|
||||
});
|
||||
// END HACK: PREPARE FOR CHANGING TO CASE-INSENSITIVE URLS
|
||||
|
||||
it('should emit the not-found document if the document is not found on the server', () => {
|
||||
let currentDocument: DocumentContents|undefined;
|
||||
|
|
@ -119,7 +107,7 @@ describe('DocumentService', () => {
|
|||
|
||||
docService.currentDocument.subscribe(doc => currentDocument = doc);
|
||||
|
||||
httpMock.expectOne({}).flush(null, { status: 404, statusText: 'NOT FOUND'});
|
||||
httpMock.expectOne({}).flush(null, {status: 404, statusText: 'NOT FOUND'});
|
||||
expect(currentDocument).toEqual(hardCodedNotFoundDoc);
|
||||
|
||||
// now check that we haven't killed the currentDocument observable sequence
|
||||
|
|
@ -143,7 +131,7 @@ describe('DocumentService', () => {
|
|||
[jasmine.any(Error)]
|
||||
]);
|
||||
expect(logger.output.error[0][0].message)
|
||||
.toEqual(`Error fetching document 'initial/doc': (Http failure response for generated/docs/initial/doc.json: 500 Server Error)`);
|
||||
.toEqual(`Error fetching document 'initial/doc': (Http failure response for generated/docs/initial/doc.json: 500 Server Error)`);
|
||||
|
||||
locationService.go('new/doc');
|
||||
httpMock.expectOne({}).flush(doc1);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { Injectable } from '@angular/core';
|
||||
import { HttpClient, HttpErrorResponse } from '@angular/common/http';
|
||||
|
||||
import { AsyncSubject, Observable, of, throwError } from 'rxjs';
|
||||
import { AsyncSubject, Observable, of } from 'rxjs';
|
||||
import { catchError, switchMap, tap } from 'rxjs/operators';
|
||||
|
||||
import { DocumentContents } from './document-contents';
|
||||
|
|
@ -53,7 +53,7 @@ export class DocumentService {
|
|||
}
|
||||
|
||||
private fetchDocument(id: string): Observable<DocumentContents> {
|
||||
const requestPath = `${DOC_CONTENT_URL_PREFIX}${id}.json`;
|
||||
const requestPath = `${DOC_CONTENT_URL_PREFIX}${encodeToLowercase(id)}.json`;
|
||||
const subject = new AsyncSubject<DocumentContents>();
|
||||
|
||||
this.logger.log('fetching document from', requestPath);
|
||||
|
|
@ -66,20 +66,6 @@ export class DocumentService {
|
|||
throw Error('Invalid data');
|
||||
}
|
||||
}),
|
||||
// HACK: PREPARE FOR CHANGING TO CASE-INSENSITIVE URLS
|
||||
catchError((error: HttpErrorResponse) => {
|
||||
const encodedPath = encodeToLowercase(requestPath);
|
||||
return error.status === 404 && encodedPath !== requestPath ?
|
||||
this.http.get<DocumentContents>(encodedPath) :
|
||||
throwError(error);
|
||||
}),
|
||||
catchError((error: HttpErrorResponse) => {
|
||||
const disambiguatedPath = convertDisambiguatedPath(requestPath);
|
||||
return error.status === 404 && disambiguatedPath !== requestPath ?
|
||||
this.http.get<DocumentContents>(disambiguatedPath) :
|
||||
throwError(error);
|
||||
}),
|
||||
// END HACK: PREPARE FOR CHANGING TO CASE-INSENSITIVE URLS
|
||||
catchError((error: HttpErrorResponse) => {
|
||||
return error.status === 404 ? this.getFileNotFoundDoc(id) : this.getErrorDoc(id, error);
|
||||
}),
|
||||
|
|
@ -123,18 +109,3 @@ export class DocumentService {
|
|||
function encodeToLowercase(str: string): string {
|
||||
return str.replace(/[A-Z_]/g, char => char.toLowerCase() + '_');
|
||||
}
|
||||
|
||||
/**
|
||||
* A temporary function to deal with a future change to URL disambiguation.
|
||||
*
|
||||
* Currently there are disambiguated URLs such as `INJECTOR-0` and `Injector-1`, which
|
||||
* will attempt to load their document contents from `injector-0.json` and `injector-1.json`
|
||||
* respectively. In a future version of the AIO app, the disambiguation will be changed to
|
||||
* escape the upper-case characters instead.
|
||||
*
|
||||
* This function will be called if the current AIO is trying to request documents from a
|
||||
* server that has been updated to use the new disambiguated URLs.
|
||||
*/
|
||||
function convertDisambiguatedPath(str: string): string {
|
||||
return encodeToLowercase(str.replace(/-\d+\.json$/, '.json'));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ describe(browser.baseUrl, () => {
|
|||
|
||||
describe('(api docs pages)', () => {
|
||||
const textPerUrl: { [key: string]: string } = {
|
||||
/* Class */ 'api/core/Injector-0': 'class injector',
|
||||
/* Class */ 'api/core/Injector': 'class injector',
|
||||
/* Const */ 'api/forms/NG_VALIDATORS': 'const ng_validators',
|
||||
/* Decorator */ 'api/core/Component': '@component',
|
||||
/* Directive */ 'api/common/NgIf': 'class ngif',
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* @dgProcessor disambiguateDocPathsProcessor
|
||||
* @description
|
||||
*
|
||||
* Ensures that docs that have the same path, other than case changes,
|
||||
* Ensures that docs that have the same output path, other than case changes,
|
||||
* are disambiguated.
|
||||
*
|
||||
* For example in Angular there is the `ROUTES` const and a `Routes` type.
|
||||
|
|
@ -14,55 +14,30 @@
|
|||
* ```
|
||||
*
|
||||
* but in a case-insensitive file-system these two paths point to the same file!
|
||||
*
|
||||
* So this processor will encode the paths into lower case that is not affected
|
||||
* by case-insensitive file-systems.
|
||||
*/
|
||||
module.exports = function disambiguateDocPathsProcessor(log) {
|
||||
module.exports = function disambiguateDocPathsProcessor() {
|
||||
return {
|
||||
$runAfter: ['paths-computed'],
|
||||
$runBefore: ['rendering-docs', 'createSitemap'],
|
||||
$process(docs) {
|
||||
// Collect all the ambiguous docs, whose outputPath is are only different by casing.
|
||||
const ambiguousDocMap = new Map();
|
||||
for (const doc of docs) {
|
||||
if (!doc.outputPath) {
|
||||
continue;
|
||||
}
|
||||
const outputPath = doc.outputPath.toLowerCase();
|
||||
if (!ambiguousDocMap.has(outputPath)) {
|
||||
ambiguousDocMap.set(outputPath, []);
|
||||
}
|
||||
const ambiguousDocs = ambiguousDocMap.get(outputPath);
|
||||
ambiguousDocs.push(doc);
|
||||
}
|
||||
|
||||
// Create a disambiguator doc for each set of such ambiguous docs,
|
||||
// and update the ambiguous docs to have unique `path` and `outputPath` properties.
|
||||
for (const [outputPath, ambiguousDocs] of ambiguousDocMap) {
|
||||
if (ambiguousDocs.length === 1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
log.debug('Docs with ambiguous outputPath:' + ambiguousDocs.map((d, i) => `\n - ${d.id}: "${d.outputPath}" replaced with "${convertPath(d.outputPath, i)}".`));
|
||||
|
||||
const doc = ambiguousDocs[0];
|
||||
const path = doc.path;
|
||||
const id = `${doc.id.toLowerCase()}-disambiguator`;
|
||||
const title = `${doc.id.toLowerCase()} (disambiguation)`;
|
||||
const aliases = [id];
|
||||
docs.push({ docType: 'disambiguator', id, title, aliases, path, outputPath, docs: ambiguousDocs });
|
||||
|
||||
// Update the paths
|
||||
let count = 0;
|
||||
for (const doc of ambiguousDocs) {
|
||||
doc.path = convertPath(doc.path, count);
|
||||
doc.outputPath = convertPath(doc.outputPath, count);
|
||||
count += 1;
|
||||
}
|
||||
doc.outputPath = encodeToLowercase(doc.outputPath);
|
||||
}
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
function convertPath(path, count) {
|
||||
// Add the counter before any extension
|
||||
return path.replace(/(\.[^.]*)?$/, `-${count}$1`);
|
||||
/**
|
||||
* To avoid collisions on case-insensitive file-systems, we encode the path to the content in
|
||||
* a deterministic case-insensitive form - converting all uppercase letters to lowercase followed
|
||||
* by an underscore.
|
||||
*/
|
||||
function encodeToLowercase(str) {
|
||||
return str.replace(/[A-Z_]/g, char => char.toLowerCase() + '_');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,12 +9,14 @@ describe('disambiguateDocPaths processor', () => {
|
|||
injector = dgeni.configureInjector();
|
||||
processor = injector.get('disambiguateDocPathsProcessor');
|
||||
docs = [
|
||||
{docType: 'test-doc', id: 'test-doc', path: 'test/doc', outputPath: 'test/doc.json'},
|
||||
{docType: 'test-doc', id: 'TEST-DOC', path: 'TEST/DOC', outputPath: 'TEST/DOC.json'},
|
||||
{docType: 'test-doc', id: 'test-Doc', path: 'test/Doc', outputPath: 'test/Doc.xml'},
|
||||
{docType: 'test-doc', id: 'unique-doc', path: 'unique/doc', outputPath: 'unique/doc.json'},
|
||||
{docType: 'test-doc', id: 'other-doc', path: 'other/doc', outputPath: 'other/doc.json'},
|
||||
{docType: 'test-doc', id: 'other-DOC', path: 'other/DOC', outputPath: 'other/DOC.json'},
|
||||
{ docType: 'test-doc', id: 'test-doc', path: 'test/doc', outputPath: 'test/doc.json' },
|
||||
{ docType: 'test-doc', id: 'TEST-DOC', path: 'TEST/DOC', outputPath: 'TEST/DOC.json' },
|
||||
{ docType: 'test-doc', id: 'test-Doc', path: 'test/Doc', outputPath: 'test/Doc.xml' },
|
||||
{ docType: 'test-doc', id: 'unique-doc', path: 'unique/doc', outputPath: 'unique/doc.json' },
|
||||
{ docType: 'test-doc', id: 'other-doc', path: 'other/doc', outputPath: 'other/doc.json' },
|
||||
{ docType: 'test-doc', id: 'other-DOC', path: 'other/DOC', outputPath: 'other/DOC.json' },
|
||||
{ docType: 'test-doc', id: 'has_underscore', path: 'has_underscore', outputPath: 'has_underscore.json' },
|
||||
{ docType: 'test-doc', id: 'HAS_UNDERSCORE', path: 'HAS_UNDERSCORE', outputPath: 'HAS_UNDERSCORE.json' },
|
||||
];
|
||||
});
|
||||
|
||||
|
|
@ -26,44 +28,23 @@ describe('disambiguateDocPaths processor', () => {
|
|||
expect(processor.$runBefore).toContain('createSitemap');
|
||||
});
|
||||
|
||||
it('should create `disambiguator` documents for docs that have ambiguous outputPaths', () => {
|
||||
const numDocs = docs.length;
|
||||
it('should update the path and outputPath properties of each doc to be unambiguous on case-insensitive file-systems', () => {
|
||||
processor.$process(docs);
|
||||
expect(docs.length).toEqual(numDocs + 2);
|
||||
expect(docs[docs.length - 2]).toEqual({
|
||||
docType: 'disambiguator',
|
||||
id: 'test-doc-disambiguator',
|
||||
title: 'test-doc (disambiguation)',
|
||||
aliases: ['test-doc-disambiguator'],
|
||||
path: 'test/doc',
|
||||
outputPath: 'test/doc.json',
|
||||
docs: [docs[0], docs[1]],
|
||||
});
|
||||
expect(docs[docs.length - 1]).toEqual({
|
||||
docType: 'disambiguator',
|
||||
id: 'other-doc-disambiguator',
|
||||
title: 'other-doc (disambiguation)',
|
||||
aliases: ['other-doc-disambiguator'],
|
||||
path: 'other/doc',
|
||||
outputPath: 'other/doc.json',
|
||||
docs: [docs[4], docs[5]],
|
||||
});
|
||||
});
|
||||
|
||||
it('should update the path and outputPath properties of each ambiguous doc', () => {
|
||||
processor.$process(docs);
|
||||
expect(docs[0].path).toEqual('test/doc-0');
|
||||
expect(docs[0].outputPath).toEqual('test/doc-0.json');
|
||||
expect(docs[1].path).toEqual('TEST/DOC-1');
|
||||
expect(docs[1].outputPath).toEqual('TEST/DOC-1.json');
|
||||
|
||||
// The non-ambiguous docs are left alone
|
||||
expect(docs[2].outputPath).toEqual('test/Doc.xml');
|
||||
expect(docs[0].path).toEqual('test/doc');
|
||||
expect(docs[0].outputPath).toEqual('test/doc.json');
|
||||
expect(docs[1].path).toEqual('TEST/DOC');
|
||||
expect(docs[1].outputPath).toEqual('t_e_s_t_/d_o_c_.json');
|
||||
expect(docs[2].path).toEqual('test/Doc');
|
||||
expect(docs[2].outputPath).toEqual('test/d_oc.xml');
|
||||
expect(docs[3].path).toEqual('unique/doc');
|
||||
expect(docs[3].outputPath).toEqual('unique/doc.json');
|
||||
|
||||
expect(docs[4].path).toEqual('other/doc-0');
|
||||
expect(docs[4].outputPath).toEqual('other/doc-0.json');
|
||||
expect(docs[5].path).toEqual('other/DOC-1');
|
||||
expect(docs[5].outputPath).toEqual('other/DOC-1.json');
|
||||
expect(docs[4].path).toEqual('other/doc');
|
||||
expect(docs[4].outputPath).toEqual('other/doc.json');
|
||||
expect(docs[5].path).toEqual('other/DOC');
|
||||
expect(docs[5].outputPath).toEqual('other/d_o_c_.json');
|
||||
expect(docs[6].path).toEqual('has_underscore');
|
||||
expect(docs[6].outputPath).toEqual('has__underscore.json');
|
||||
expect(docs[7].path).toEqual('HAS_UNDERSCORE');
|
||||
expect(docs[7].outputPath).toEqual('h_a_s___u_n_d_e_r_s_c_o_r_e_.json');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -71,14 +71,14 @@ describe('authors-package (integration tests)', () => {
|
|||
it('should generate API doc if the "fileChanged" is an API doc', () => {
|
||||
return generateDocs('packages/forms/src/form_builder.ts', { silent: true }).then(() => {
|
||||
expect(fs.writeFile).toHaveBeenCalled();
|
||||
expect(files).toContain(resolve(DOCS_OUTPUT_PATH, 'api/forms/FormBuilder.json'));
|
||||
expect(files).toContain(resolve(DOCS_OUTPUT_PATH, 'api/forms/f_ormb_uilder.json'));
|
||||
});
|
||||
}, 16000);
|
||||
|
||||
it('should generate API doc if the "fileChanged" is an API example', () => {
|
||||
return generateDocs('packages/examples/forms/ts/formBuilder/form_builder_example.ts', { silent: true }).then(() => {
|
||||
expect(fs.writeFile).toHaveBeenCalled();
|
||||
expect(files).toContain(resolve(DOCS_OUTPUT_PATH, 'api/forms/FormBuilder.json'));
|
||||
expect(files).toContain(resolve(DOCS_OUTPUT_PATH, 'api/forms/f_ormb_uilder.json'));
|
||||
});
|
||||
}, 16000);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
"master": {
|
||||
"uncompressed": {
|
||||
"runtime-es2017": 4619,
|
||||
"main-es2017": 456795,
|
||||
"main-es2017": 456578,
|
||||
"polyfills-es2017": 55210
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue