test: run unit tests in random order (#19904)

PR Close #19904

karma-js.conf.js

@@ -6,8 +6,9 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-var browserProvidersConf = require('./browser-providers.conf.js');
+const browserProvidersConf = require('./browser-providers.conf');
-var internalAngularReporter = require('./tools/karma/reporter.js');
+const {generateSeed} = require('./tools/jasmine-seed-generator');
+const internalAngularReporter = require('./tools/karma/reporter');
 
 // Karma configuration
 // Generated on Thu Sep 25 2014 11:52:02 GMT-0700 (PDT)
@@ -16,6 +17,13 @@ module.exports = function(config) {
 
     frameworks: ['jasmine'],
 
+    client: {
+      jasmine: {
+        random: true,
+        seed: generateSeed('karma-js.conf'),
+      },
+    },
+
     files: [
       // Sources and specs.
       // Loaded through the System loader, in `test-main.js`.

package.json

@@ -91,7 +91,7 @@
     "karma": "0.13.20",
     "karma-browserstack-launcher": "0.1.9",
     "karma-chrome-launcher": "0.2.0",
-    "karma-jasmine": "0.3.6",
+    "karma-jasmine": "^1.1.2",
     "karma-sauce-launcher": "0.3.0",
     "karma-sourcemap-loader": "0.3.6",
     "madge": "0.5.0",

packages/elements/test/component-factory-strategy_spec.ts

@@ -26,6 +26,8 @@ describe('ComponentFactoryNgElementStrategy', () => {
     componentRef = factory.componentRef;
 
     applicationRef = jasmine.createSpyObj('applicationRef', ['attachView']);
+    injector = jasmine.createSpyObj('injector', ['get']);
+    injector.get.and.returnValue(applicationRef);
 
     strategy = new ComponentNgElementStrategy(factory, injector);
   });
@@ -33,7 +35,6 @@ describe('ComponentFactoryNgElementStrategy', () => {
   it('should create a new strategy from the factory', () => {
     const factoryResolver = jasmine.createSpyObj('factoryResolver', ['resolveComponentFactory']);
     factoryResolver.resolveComponentFactory.and.returnValue(factory);
-    injector = jasmine.createSpyObj('injector', ['get']);
     injector.get.and.returnValue(factoryResolver);
 
     const strategyFactory = new ComponentNgElementStrategyFactory(FakeComponent, injector);
@@ -44,7 +45,6 @@ describe('ComponentFactoryNgElementStrategy', () => {
     beforeEach(() => {
       // Set up an initial value to make sure it is passed to the component
       strategy.setInputValue('fooFoo', 'fooFoo-1');
-      injector.get.and.returnValue(applicationRef);
       strategy.connect(document.createElement('div'));
     });
 

packages/forms/test/reactive_integration_spec.ts

@@ -8,7 +8,7 @@
 
 import {Component, Directive, Input, Type, forwardRef} from '@angular/core';
 import {ComponentFixture, TestBed, fakeAsync, tick} from '@angular/core/testing';
-import {AbstractControl, AsyncValidator, AsyncValidatorFn, COMPOSITION_BUFFER_MODE, FormArray, FormControl, FormGroup, FormGroupDirective, FormsModule, NG_ASYNC_VALIDATORS, NG_VALIDATORS, ReactiveFormsModule, Validators} from '@angular/forms';
+import {AbstractControl, AsyncValidator, AsyncValidatorFn, COMPOSITION_BUFFER_MODE, FormArray, FormControl, FormControlDirective, FormControlName, FormGroup, FormGroupDirective, FormsModule, NG_ASYNC_VALIDATORS, NG_VALIDATORS, ReactiveFormsModule, Validators} from '@angular/forms';
 import {By} from '@angular/platform-browser/src/dom/debug/by';
 import {getDOM} from '@angular/platform-browser/src/dom/dom_adapter';
 import {dispatchEvent} from '@angular/platform-browser/testing/src/browser_util';
@@ -1625,14 +1625,17 @@ import {MyInput, MyInputForm} from './value_accessor_integration_spec';
     });
 
     describe('ngModel interactions', () => {
+      let warnSpy: jasmine.Spy;
+
+      beforeEach(() => {
+        // Reset `_ngModelWarningSentOnce` on `FormControlDirective` and `FormControlName` types.
+        (FormControlDirective as any)._ngModelWarningSentOnce = false;
+        (FormControlName as any)._ngModelWarningSentOnce = false;
+
+        warnSpy = spyOn(console, 'warn');
+      });
 
       describe('deprecation warnings', () => {
-        let warnSpy: any;
-
-        beforeEach(() => {
-          warnSpy = jasmine.createSpy('warn');
-          console.warn = warnSpy;
-        });
 
         it('should warn once by default when using ngModel with formControlName', fakeAsync(() => {
              const fixture = initTest(FormGroupNgModel);

packages/language-service/test/diagnostics_spec.ts

@@ -9,19 +9,25 @@
 import * as ts from 'typescript';
 
 import {createLanguageService} from '../src/language_service';
-import {Diagnostics} from '../src/types';
+import {Diagnostics, LanguageService} from '../src/types';
 import {TypeScriptServiceHost} from '../src/typescript_host';
 
 import {toh} from './test_data';
 import {MockTypescriptHost, diagnosticMessageContains, findDiagnostic, includeDiagnostic, noDiagnostics} from './test_utils';
 
 describe('diagnostics', () => {
-  let documentRegistry = ts.createDocumentRegistry();
+  let mockHost: MockTypescriptHost;
-  let mockHost = new MockTypescriptHost(['/app/main.ts', '/app/parsing-cases.ts'], toh);
+  let ngHost: TypeScriptServiceHost;
-  let service = ts.createLanguageService(mockHost, documentRegistry);
+  let ngService: LanguageService;
-  let ngHost = new TypeScriptServiceHost(mockHost, service);
+
-  let ngService = createLanguageService(ngHost);
+  beforeEach(() => {
-  ngHost.setSite(ngService);
+    mockHost = new MockTypescriptHost(['/app/main.ts', '/app/parsing-cases.ts'], toh);
+    const documentRegistry = ts.createDocumentRegistry();
+    const service = ts.createLanguageService(mockHost, documentRegistry);
+    ngHost = new TypeScriptServiceHost(mockHost, service);
+    ngService = createLanguageService(ngHost);
+    ngHost.setSite(ngService);
+  });
 
   it('should be no diagnostics for test.ng',
      () => { expect(ngService.getDiagnostics('/app/test.ng')).toEqual([]); });
@@ -99,7 +105,7 @@ describe('diagnostics', () => {
       const code = '\n@Component({template: \'<form></form>\'}) export class MyComponent {}';
       addCode(code, (fileName, content) => {
         const diagnostics = ngService.getDiagnostics(fileName);
-        expectOnlyModuleDiagnostics(diagnostics !);
+        expectOnlyModuleDiagnostics(diagnostics);
       });
     });
 
@@ -139,7 +145,7 @@ describe('diagnostics', () => {
           ` @Component({template: \`<div *ngIf="something === 'foo'"></div>\`}) export class MyComponent { something: 'foo' | 'bar'; }`;
       addCode(code, fileName => {
         const diagnostics = ngService.getDiagnostics(fileName);
-        expectOnlyModuleDiagnostics(diagnostics !);
+        expectOnlyModuleDiagnostics(diagnostics);
       });
     });
 
@@ -160,7 +166,7 @@ describe('diagnostics', () => {
           ` @Component({template: \`<div *ngIf="something === undefined"></div>\`}) export class MyComponent { something = 'foo'; }})`;
       addCode(code, fileName => {
         const diagnostics = ngService.getDiagnostics(fileName);
-        expectOnlyModuleDiagnostics(diagnostics !);
+        expectOnlyModuleDiagnostics(diagnostics);
       });
     });
 
@@ -253,7 +259,7 @@ describe('diagnostics', () => {
         })
         export class MyComponent {}
       `,
-          fileName => expectOnlyModuleDiagnostics(ngService.getDiagnostics(fileName) !));
+          fileName => expectOnlyModuleDiagnostics(ngService.getDiagnostics(fileName)));
     });
 
     // Issue #15625
@@ -273,7 +279,7 @@ describe('diagnostics', () => {
           }
       `,
           fileName => {
-            const diagnostics = ngService.getDiagnostics(fileName) !;
+            const diagnostics = ngService.getDiagnostics(fileName);
             expectOnlyModuleDiagnostics(diagnostics);
           });
     });
@@ -368,14 +374,17 @@ describe('diagnostics', () => {
     function expectOnlyModuleDiagnostics(diagnostics: Diagnostics | undefined) {
       // Expect only the 'MyComponent' diagnostic
       if (!diagnostics) throw new Error('Expecting Diagnostics');
-      expect(diagnostics.length).toBe(1);
       if (diagnostics.length > 1) {
-        for (const diagnostic of diagnostics) {
+        const unexpectedDiagnostics =
-          if (diagnosticMessageContains(diagnostic.message, 'MyComponent')) continue;
+            diagnostics.filter(diag => !diagnosticMessageContains(diag.message, 'MyComponent'))
-          fail(`(${diagnostic.span.start}:${diagnostic.span.end}): ${diagnostic.message}`);
+                .map(diag => `(${diag.span.start}:${diag.span.end}): ${diag.message}`);
+
+        if (unexpectedDiagnostics.length) {
+          fail(`Unexpected diagnostics:\n  ${unexpectedDiagnostics.join('\n  ')}`);
+          return;
         }
-        return;
       }
+      expect(diagnostics.length).toBe(1);
       expect(diagnosticMessageContains(diagnostics[0].message, 'MyComponent')).toBeTruthy();
     }
   });

packages/router/karma.conf.js

@@ -6,7 +6,8 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-var browserProvidersConf = require('../../browser-providers.conf.js');
+const browserProvidersConf = require('../../browser-providers.conf');
+const {generateSeed} = require('../../tools/jasmine-seed-generator');
 
 // Karma configuration
 module.exports = function(config) {
@@ -16,6 +17,13 @@ module.exports = function(config) {
 
     frameworks: ['jasmine'],
 
+    client: {
+      jasmine: {
+        random: true,
+        seed: generateSeed('router/karma.conf'),
+      },
+    },
+
     files: [
       // Polyfills.
       'node_modules/core-js/client/core.js',

packages/upgrade/test/common/downgrade_component_adapter_spec.ts

@@ -5,8 +5,8 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
-import {ApplicationRef, Compiler, Component, ComponentFactory, ComponentRef, Injector, NgModule, Testability, TestabilityRegistry} from '@angular/core';
+import {Compiler, Component, ComponentFactory, Injector, NgModule, TestabilityRegistry} from '@angular/core';
-import {TestBed, getTestBed, inject} from '@angular/core/testing';
+import {TestBed} from '@angular/core/testing';
 import * as angular from '@angular/upgrade/src/common/angular1';
 import {DowngradeComponentAdapter, groupNodesBySelector} from '@angular/upgrade/src/common/downgrade_component_adapter';
 
@@ -82,6 +82,7 @@ withEachNg1Version(() => {
       let adapter: DowngradeComponentAdapter;
       let content: string;
       let compiler: Compiler;
+      let registry: TestabilityRegistry;
       let element: angular.IAugmentedJQuery;
 
       class mockScope implements angular.IScope {
@@ -168,15 +169,13 @@ withEachNg1Version(() => {
             componentFactory, wrapCallback);
       }
 
-      beforeEach((inject([Compiler], (inject_compiler: Compiler) => {
+      beforeEach(() => {
-        compiler = inject_compiler;
+        compiler = TestBed.get(Compiler);
+        registry = TestBed.get(TestabilityRegistry);
         adapter = getAdaptor();
-      })));
-
-      afterEach(() => {
-        let registry = TestBed.get(TestabilityRegistry);
-        registry.unregisterAllApplications();
       });
+      beforeEach(() => registry.unregisterAllApplications());
+      afterEach(() => registry.unregisterAllApplications());
 
       it('should add testabilities hook when creating components', () => {
 

tools/cjs-jasmine/index-tools.ts

@@ -18,6 +18,7 @@ require('zone.js/dist/proxy.js');
 require('zone.js/dist/sync-test.js');
 require('zone.js/dist/async-test.js');
 require('zone.js/dist/fake-async-test.js');
+const {generateSeed} = require('../../../tools/jasmine-seed-generator');
 
 // Let TypeScript know this is a module
 export {};
@@ -32,8 +33,10 @@ require('zone.js/dist/jasmine-patch.js');
 // Config the test runner
 jrunner.jasmine.DEFAULT_TIMEOUT_INTERVAL = 100;
 jrunner.loadConfig({
+  random: true,
   spec_dir: '',
 });
+jrunner.seed(generateSeed('cjs-jasmine/index-tools'));
 jrunner.configureDefaultReporter({showColors: process.argv.indexOf('--no-color') === -1});
 jrunner.onComplete((passed: boolean) => process.exit(passed ? 0 : 1));
 

tools/cjs-jasmine/index.ts

@@ -20,6 +20,7 @@ require('zone.js/dist/sync-test.js');
 require('zone.js/dist/async-test.js');
 require('zone.js/dist/fake-async-test.js');
 require('reflect-metadata/Reflect');
+const {generateSeed} = require('../../../tools/jasmine-seed-generator');
 
 // Let TypeScript know this is a module
 export {};
@@ -37,8 +38,10 @@ require('zone.js/dist/jasmine-patch.js');
 // Config the test runner
 jrunner.jasmine.DEFAULT_TIMEOUT_INTERVAL = 100;
 jrunner.loadConfig({
+  random: true,
   spec_dir: '',
 });
+jrunner.seed(generateSeed('cjs-jasmine/index'));
 jrunner.configureDefaultReporter({showColors: process.argv.indexOf('--no-color') === -1});
 jrunner.onComplete((passed: boolean) => process.exit(passed ? 0 : 1));
 

tools/jasmine-seed-generator.js

@@ -0,0 +1,19 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+'use strict';
+
+// Generate a "random" seed, suitable to be used for pseudo-randomizing jasmine tests.
+module.exports = {
+  generateSeed: caller => {
+    const seed = process.env.JASMINE_RANDOM_SEED || String(Math.random()).slice(-5);
+    // tslint:disable-next-line: no-console
+    console.log(`[${caller}] Jasmine random seed: ${seed}`);
+    return seed;
+  },
+};

yarn.lock

@@ -3492,9 +3492,9 @@ karma-chrome-launcher@0.2.0:
     fs-access "^1.0.0"
     which "^1.0.9"
 
-karma-jasmine@0.3.6:
+karma-jasmine@^1.1.2:
-  version "0.3.6"
+  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/karma-jasmine/-/karma-jasmine-0.3.6.tgz#ddcc34413ac0405aa34ce29ff472e957420b857a"
+  resolved "https://registry.yarnpkg.com/karma-jasmine/-/karma-jasmine-1.1.2.tgz#394f2b25ffb4a644b9ada6f22d443e2fd08886c3"
 
 karma-sauce-launcher@0.3.0:
   version "0.3.0"