build(docs-infra): use case-insensitive encoding for content files (#42414)

To avoid having content files that have the same file path on case-insensitive
file-systems, we now encode the paths to remove uppercase characters.

PR Close #42414

aio/src/app/documents/document.service.spec.ts

@@ -67,29 +67,17 @@ describe('DocumentService', () => {
       expect(latestDocument).toEqual(doc1);
     });
 
-    // HACK: PREPARE FOR CHANGING TO CASE-INSENSITIVE URLS
+    it('should encode the request path to be case-insensitive', () => {
-    it('should attempt disambiguated document paths if the document is not found on the server', () => {
+      const { docService, locationService } = getServices('initial/Doc');
-      let currentDocument: DocumentContents|undefined;
+      docService.currentDocument.subscribe();
-      const notFoundDoc = { id: FILE_NOT_FOUND_ID, contents: '<h1>Page Not Found</h1>' };
+      httpMock.expectOne(CONTENT_URL_PREFIX + 'initial/d_oc.json').flush({});
-      const { docService, logger } = getServices('missing/Doc-1');
+      locationService.go('NEW/Doc');
-      docService.currentDocument.subscribe(doc => currentDocument = doc);
+      httpMock.expectOne(CONTENT_URL_PREFIX + 'n_e_w_/d_oc.json').flush({});
-
+      locationService.go('doc_with_underscores');
-      // Initial request return 404.
+      httpMock.expectOne(CONTENT_URL_PREFIX + 'doc__with__underscores.json').flush({});
-      httpMock.expectOne({url: 'generated/docs/missing/Doc-1.json'}).flush(null, {status: 404, statusText: 'NOT FOUND'});
+      locationService.go('DOC_WITH_UNDERSCORES');
-      httpMock.expectOne({url: 'generated/docs/missing/d_oc-1.json'}).flush(null, {status: 404, statusText: 'NOT FOUND'});
+      httpMock.expectOne(CONTENT_URL_PREFIX + 'd_o_c___w_i_t_h___u_n_d_e_r_s_c_o_r_e_s_.json').flush({});
-      httpMock.expectOne({url: 'generated/docs/missing/d_oc.json'}).flush(null, {status: 404, statusText: 'NOT FOUND'});
-      expect(logger.output.error).toEqual([
-        [jasmine.any(Error)]
-      ]);
-      expect(logger.output.error[0][0].message).toEqual(`Document file not found at 'missing/Doc-1'`);
-
-      // Subsequent request for not-found document.
-      logger.output.error = [];
-      httpMock.expectOne(CONTENT_URL_PREFIX + 'file-not-found.json').flush(notFoundDoc);
-      expect(logger.output.error).toEqual([]); // does not report repeated errors
-      expect(currentDocument).toEqual(notFoundDoc);
     });
-    // END HACK: PREPARE FOR CHANGING TO CASE-INSENSITIVE URLS
 
     it('should emit the not-found document if the document is not found on the server', () => {
       let currentDocument: DocumentContents|undefined;

aio/src/app/documents/document.service.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@angular/core';
 import { HttpClient, HttpErrorResponse } from '@angular/common/http';
 
-import { AsyncSubject, Observable, of, throwError } from 'rxjs';
+import { AsyncSubject, Observable, of } from 'rxjs';
 import { catchError, switchMap, tap } from 'rxjs/operators';
 
 import { DocumentContents } from './document-contents';
@@ -53,7 +53,7 @@ export class DocumentService {
   }
 
   private fetchDocument(id: string): Observable<DocumentContents> {
-    const requestPath = `${DOC_CONTENT_URL_PREFIX}${id}.json`;
+    const requestPath = `${DOC_CONTENT_URL_PREFIX}${encodeToLowercase(id)}.json`;
     const subject = new AsyncSubject<DocumentContents>();
 
     this.logger.log('fetching document from', requestPath);
@@ -66,20 +66,6 @@ export class DocumentService {
             throw Error('Invalid data');
           }
         }),
-        // HACK: PREPARE FOR CHANGING TO CASE-INSENSITIVE URLS
-        catchError((error: HttpErrorResponse) => {
-          const encodedPath = encodeToLowercase(requestPath);
-          return error.status === 404 && encodedPath !== requestPath ?
-              this.http.get<DocumentContents>(encodedPath) :
-              throwError(error);
-        }),
-        catchError((error: HttpErrorResponse) => {
-          const disambiguatedPath = convertDisambiguatedPath(requestPath);
-          return error.status === 404 && disambiguatedPath !== requestPath ?
-              this.http.get<DocumentContents>(disambiguatedPath) :
-              throwError(error);
-        }),
-        // END HACK: PREPARE FOR CHANGING TO CASE-INSENSITIVE URLS
         catchError((error: HttpErrorResponse) => {
           return error.status === 404 ? this.getFileNotFoundDoc(id) : this.getErrorDoc(id, error);
         }),
@@ -123,18 +109,3 @@ export class DocumentService {
 function encodeToLowercase(str: string): string {
   return str.replace(/[A-Z_]/g, char => char.toLowerCase() + '_');
 }
-
-/**
- * A temporary function to deal with a future change to URL disambiguation.
- *
- * Currently there are disambiguated URLs such as `INJECTOR-0` and `Injector-1`, which
- * will attempt to load their document contents from `injector-0.json` and `injector-1.json`
- * respectively. In a future version of the AIO app, the disambiguation will be changed to
- * escape the upper-case characters instead.
- *
- * This function will be called if the current AIO is trying to request documents from a
- * server that has been updated to use the new disambiguated URLs.
- */
-function convertDisambiguatedPath(str: string): string {
-  return encodeToLowercase(str.replace(/-\d+\.json$/, '.json'));
-}

aio/tests/deployment/e2e/smoke-tests.e2e-spec.ts

@@ -62,7 +62,7 @@ describe(browser.baseUrl, () => {
 
     describe('(api docs pages)', () => {
       const textPerUrl: { [key: string]: string } = {
-        /* Class */ 'api/core/Injector-0': 'class injector',
+        /* Class */ 'api/core/Injector': 'class injector',
         /* Const */ 'api/forms/NG_VALIDATORS': 'const ng_validators',
         /* Decorator */ 'api/core/Component': '@component',
         /* Directive */ 'api/common/NgIf': 'class ngif',

aio/tools/transforms/angular-base-package/processors/disambiguateDocPaths.js

@@ -2,7 +2,7 @@
  * @dgProcessor disambiguateDocPathsProcessor
  * @description
  *
- * Ensures that docs that have the same path, other than case changes,
+ * Ensures that docs that have the same output path, other than case changes,
  * are disambiguated.
  *
  * For example in Angular there is the `ROUTES` const and a `Routes` type.
@@ -14,55 +14,30 @@
  * ```
  *
  * but in a case-insensitive file-system these two paths point to the same file!
+ *
+ * So this processor will encode the paths into lower case that is not affected
+ * by case-insensitive file-systems.
  */
-module.exports = function disambiguateDocPathsProcessor(log) {
+module.exports = function disambiguateDocPathsProcessor() {
   return {
     $runAfter: ['paths-computed'],
     $runBefore: ['rendering-docs', 'createSitemap'],
     $process(docs) {
-      // Collect all the ambiguous docs, whose outputPath is are only different by casing.
-      const ambiguousDocMap = new Map();
       for (const doc of docs) {
         if (!doc.outputPath) {
           continue;
         }
-        const outputPath = doc.outputPath.toLowerCase();
+        doc.outputPath = encodeToLowercase(doc.outputPath);
-        if (!ambiguousDocMap.has(outputPath)) {
-          ambiguousDocMap.set(outputPath, []);
-        }
-        const ambiguousDocs = ambiguousDocMap.get(outputPath);
-        ambiguousDocs.push(doc);
-      }
-
-      // Create a disambiguator doc for each set of such ambiguous docs,
-      // and update the ambiguous docs to have unique `path` and `outputPath` properties.
-      for (const [outputPath, ambiguousDocs] of ambiguousDocMap) {
-        if (ambiguousDocs.length === 1) {
-          continue;
-        }
-
-        log.debug('Docs with ambiguous outputPath:' + ambiguousDocs.map((d, i) => `\n - ${d.id}: "${d.outputPath}" replaced with "${convertPath(d.outputPath, i)}".`));
-
-        const doc = ambiguousDocs[0];
-        const path = doc.path;
-        const id = `${doc.id.toLowerCase()}-disambiguator`;
-        const title = `${doc.id.toLowerCase()} (disambiguation)`;
-        const aliases = [id];
-        docs.push({ docType: 'disambiguator', id, title, aliases, path, outputPath, docs: ambiguousDocs });
-
-        // Update the paths
-        let count = 0;
-        for (const doc of ambiguousDocs) {
-          doc.path = convertPath(doc.path, count);
-          doc.outputPath = convertPath(doc.outputPath, count);
-          count += 1;
-        }
       }
     }
   };
 };
 
-function convertPath(path, count) {
+/**
-  // Add the counter before any extension
+ * To avoid collisions on case-insensitive file-systems, we encode the path to the content in
-  return path.replace(/(\.[^.]*)?$/, `-${count}$1`);
+ * a deterministic case-insensitive form - converting all uppercase letters to lowercase followed
+ * by an underscore.
+ */
+function encodeToLowercase(str) {
+  return str.replace(/[A-Z_]/g, char => char.toLowerCase() + '_');
 }

aio/tools/transforms/angular-base-package/processors/disambiguateDocPaths.spec.js

@@ -15,6 +15,8 @@ describe('disambiguateDocPaths processor', () => {
       { docType: 'test-doc', id: 'unique-doc', path: 'unique/doc', outputPath: 'unique/doc.json' },
       { docType: 'test-doc', id: 'other-doc', path: 'other/doc', outputPath: 'other/doc.json' },
       { docType: 'test-doc', id: 'other-DOC', path: 'other/DOC', outputPath: 'other/DOC.json' },
+      { docType: 'test-doc', id: 'has_underscore', path: 'has_underscore', outputPath: 'has_underscore.json' },
+      { docType: 'test-doc', id: 'HAS_UNDERSCORE', path: 'HAS_UNDERSCORE', outputPath: 'HAS_UNDERSCORE.json' },
     ];
   });
 
@@ -26,44 +28,23 @@ describe('disambiguateDocPaths processor', () => {
     expect(processor.$runBefore).toContain('createSitemap');
   });
 
-  it('should create `disambiguator` documents for docs that have ambiguous outputPaths', () => {
+  it('should update the path and outputPath properties of each doc to be unambiguous on case-insensitive file-systems', () => {
-    const numDocs = docs.length;
     processor.$process(docs);
-    expect(docs.length).toEqual(numDocs + 2);
+    expect(docs[0].path).toEqual('test/doc');
-    expect(docs[docs.length - 2]).toEqual({
+    expect(docs[0].outputPath).toEqual('test/doc.json');
-      docType: 'disambiguator',
+    expect(docs[1].path).toEqual('TEST/DOC');
-      id: 'test-doc-disambiguator',
+    expect(docs[1].outputPath).toEqual('t_e_s_t_/d_o_c_.json');
-      title: 'test-doc (disambiguation)',
+    expect(docs[2].path).toEqual('test/Doc');
-      aliases: ['test-doc-disambiguator'],
+    expect(docs[2].outputPath).toEqual('test/d_oc.xml');
-      path: 'test/doc',
+    expect(docs[3].path).toEqual('unique/doc');
-      outputPath: 'test/doc.json',
-      docs: [docs[0], docs[1]],
-    });
-    expect(docs[docs.length - 1]).toEqual({
-      docType: 'disambiguator',
-      id: 'other-doc-disambiguator',
-      title: 'other-doc (disambiguation)',
-      aliases: ['other-doc-disambiguator'],
-      path: 'other/doc',
-      outputPath: 'other/doc.json',
-      docs: [docs[4], docs[5]],
-    });
-  });
-
-  it('should update the path and outputPath properties of each ambiguous doc', () => {
-    processor.$process(docs);
-    expect(docs[0].path).toEqual('test/doc-0');
-    expect(docs[0].outputPath).toEqual('test/doc-0.json');
-    expect(docs[1].path).toEqual('TEST/DOC-1');
-    expect(docs[1].outputPath).toEqual('TEST/DOC-1.json');
-
-    // The non-ambiguous docs are left alone
-    expect(docs[2].outputPath).toEqual('test/Doc.xml');
     expect(docs[3].outputPath).toEqual('unique/doc.json');
-
+    expect(docs[4].path).toEqual('other/doc');
-    expect(docs[4].path).toEqual('other/doc-0');
+    expect(docs[4].outputPath).toEqual('other/doc.json');
-    expect(docs[4].outputPath).toEqual('other/doc-0.json');
+    expect(docs[5].path).toEqual('other/DOC');
-    expect(docs[5].path).toEqual('other/DOC-1');
+    expect(docs[5].outputPath).toEqual('other/d_o_c_.json');
-    expect(docs[5].outputPath).toEqual('other/DOC-1.json');
+    expect(docs[6].path).toEqual('has_underscore');
+    expect(docs[6].outputPath).toEqual('has__underscore.json');
+    expect(docs[7].path).toEqual('HAS_UNDERSCORE');
+    expect(docs[7].outputPath).toEqual('h_a_s___u_n_d_e_r_s_c_o_r_e_.json');
   });
 });

aio/tools/transforms/authors-package/index.spec.js

@@ -71,14 +71,14 @@ describe('authors-package (integration tests)', () => {
   it('should generate API doc if the "fileChanged" is an API doc', () => {
     return generateDocs('packages/forms/src/form_builder.ts', { silent: true }).then(() => {
       expect(fs.writeFile).toHaveBeenCalled();
-      expect(files).toContain(resolve(DOCS_OUTPUT_PATH, 'api/forms/FormBuilder.json'));
+      expect(files).toContain(resolve(DOCS_OUTPUT_PATH, 'api/forms/f_ormb_uilder.json'));
     });
   }, 16000);
 
   it('should generate API doc if the "fileChanged" is an API example', () => {
     return generateDocs('packages/examples/forms/ts/formBuilder/form_builder_example.ts', { silent: true }).then(() => {
       expect(fs.writeFile).toHaveBeenCalled();
-      expect(files).toContain(resolve(DOCS_OUTPUT_PATH, 'api/forms/FormBuilder.json'));
+      expect(files).toContain(resolve(DOCS_OUTPUT_PATH, 'api/forms/f_ormb_uilder.json'));
     });
   }, 16000);
 });

goldens/size-tracking/aio-payloads.json

@@ -3,7 +3,7 @@
     "master": {
       "uncompressed": {
         "runtime-es2017": 4619,
-        "main-es2017": 456795,
+        "main-es2017": 456578,
         "polyfills-es2017": 55210
       }
     }