ci: do not run benchmark measurements in circleci (#34753)

Currently we run all benchmark perf tests in CircleCI. Since we do not
collect any results, we unnecessarily waste CI/RBE resources. Instead,
we should just not run benchmark perf tests in CI, but still run the
functionality e2e tests which ensure that benchmarks are not broken.

We can do this by splitting the perf and e2e tests into separate
files/targets.

PR Close #34753

.gitignore

@@ -3,7 +3,6 @@
 /dist/
 /bazel-out
 /integration/bazel/bazel-*
-e2e_test.*
 *.log
 node_modules
 

modules/benchmarks/benchmark_test.bzl

@@ -3,7 +3,8 @@ load("//tools:defaults.bzl", "protractor_web_test_suite")
 """
   Macro that can be used to define a benchmark test. This differentiates from
   a normal Protractor test suite because we specify a custom "perf" configuration
-  that sets up "@angular/benchpress".
+  that sets up "@angular/benchpress". Benchmark test targets will not run on CI
+  unless explicitly requested.
 """
 
 def benchmark_test(name, server, deps, tags = []):
@@ -15,7 +16,9 @@ def benchmark_test(name, server, deps, tags = []):
         ],
         on_prepare = "//modules/benchmarks:start-server.js",
         server = server,
-        tags = tags,
+        # Benchmark targets should not run on CI by default.
+        tags = tags + ["manual"],
+        test_suite_tags = ["manual"],
         deps = [
             "@npm//yargs",
         ] + deps,

modules/benchmarks/e2e_test.bzl

@@ -0,0 +1,17 @@
+load("//tools:defaults.bzl", "protractor_web_test_suite")
+
+"""
+  Macro that can be used to define a e2e test in `modules/benchmarks`. Targets created through
+  this macro differentiate from a "benchmark_test" as they will run on CI and do not run
+  with `@angular/benchpress`.
+"""
+
+def e2e_test(name, server, deps, **kwargs):
+    protractor_web_test_suite(
+        name = name,
+        on_prepare = "//modules/benchmarks:start-server.js",
+        server = server,
+        # `yargs` is needed as runtime dependency for the e2e utils.
+        deps = ["@npm//yargs"] + deps,
+        **kwargs
+    )

modules/benchmarks/src/largeform/BUILD.bazel

@@ -3,10 +3,21 @@ package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 load("//tools:defaults.bzl", "ts_library")
 
 ts_library(
-    name = "tests_lib",
+    name = "perf_tests_lib",
     testonly = 1,
     srcs = ["largeform_perf.spec.ts"],
     tsconfig = "//modules/benchmarks:tsconfig-e2e.json",
+    deps = [
+        "//modules/e2e_util",
+        "@npm//protractor",
+    ],
+)
+
+ts_library(
+    name = "e2e_tests_lib",
+    testonly = 1,
+    srcs = ["largeform.spec.ts"],
+    tsconfig = "//modules/benchmarks:tsconfig-e2e.json",
     deps = [
         "//modules/e2e_util",
         "@npm//@types/jasminewd2",

modules/benchmarks/src/largeform/largeform.spec.ts

@@ -0,0 +1,29 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {$, By, element} from 'protractor';
+
+import {openBrowser, verifyNoBrowserErrors} from '../../../e2e_util/e2e_util';
+
+describe('largeform benchmark', () => {
+
+  afterEach(verifyNoBrowserErrors);
+
+  it('should work for ng2', () => {
+    openBrowser({
+      url: '/',
+      params: [{name: 'copies', value: 1}],
+      ignoreBrowserSynchronization: true,
+    });
+    $('#createDom').click();
+    expect(element.all(By.css('input[name=value0]')).get(0).getAttribute('value'))
+        .toBe('someValue0');
+    $('#destroyDom').click();
+    expect(element.all(By.css('input[name=value0]')).count()).toBe(0);
+  });
+});

modules/benchmarks/src/largeform/largeform_perf.spec.ts

@@ -6,9 +6,9 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-import {$, By, element} from 'protractor';
+import {$} from 'protractor';
 
-import {openBrowser, verifyNoBrowserErrors} from '../../../e2e_util/e2e_util';
+import {verifyNoBrowserErrors} from '../../../e2e_util/e2e_util';
 import {runBenchmark} from '../../../e2e_util/perf_util';
 
 interface Worker {
@@ -29,19 +29,6 @@ describe('largeform benchmark spec', () => {
 
   afterEach(verifyNoBrowserErrors);
 
-  it('should work for ng2', () => {
-    openBrowser({
-      url: '/',
-      params: [{name: 'copies', value: 1}],
-      ignoreBrowserSynchronization: true,
-    });
-    $('#createDom').click();
-    expect(element.all(By.css('input[name=value0]')).get(0).getAttribute('value'))
-        .toBe('someValue0');
-    $('#destroyDom').click();
-    expect(element.all(By.css('input[name=value0]')).count()).toBe(0);
-  });
-
   [CreateAndDestroyWorker].forEach((worker) => {
     describe(worker.id, () => {
       it('should run for ng2', done => {

modules/benchmarks/src/largeform/ng2/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ng_module", "ts_devserver")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -38,5 +39,11 @@ ts_devserver(
 benchmark_test(
     name = "perf",
     server = ":devserver",
-    deps = ["//modules/benchmarks/src/largeform:tests_lib"],
+    deps = ["//modules/benchmarks/src/largeform:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = ["//modules/benchmarks/src/largeform:e2e_tests_lib"],
 )

modules/benchmarks/src/largetable/BUILD.bazel

@@ -9,7 +9,7 @@ ts_library(
 )
 
 ts_library(
-    name = "perf_lib",
+    name = "perf_tests_lib",
     testonly = 1,
     srcs = ["largetable_perf.spec.ts"],
     deps = [
@@ -17,3 +17,14 @@ ts_library(
         "@npm//protractor",
     ],
 )
+
+ts_library(
+    name = "e2e_tests_lib",
+    testonly = 1,
+    srcs = ["largetable.spec.ts"],
+    tsconfig = "//modules/benchmarks:tsconfig-e2e.json",
+    deps = [
+        "//modules/e2e_util",
+        "@npm//protractor",
+    ],
+)

modules/benchmarks/src/largetable/baseline/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ts_devserver", "ts_library")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -24,5 +25,11 @@ ts_devserver(
 benchmark_test(
     name = "perf",
     server = ":devserver",
-    deps = ["//modules/benchmarks/src/largetable:perf_lib"],
+    deps = ["//modules/benchmarks/src/largetable:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = ["//modules/benchmarks/src/largetable:e2e_tests_lib"],
 )

modules/benchmarks/src/largetable/incremental_dom/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ts_devserver", "ts_library")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -27,5 +28,11 @@ ts_devserver(
 benchmark_test(
     name = "perf",
     server = ":devserver",
-    deps = ["//modules/benchmarks/src/largetable:perf_lib"],
+    deps = ["//modules/benchmarks/src/largetable:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = ["//modules/benchmarks/src/largetable:e2e_tests_lib"],
 )

modules/benchmarks/src/largetable/iv/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ts_devserver")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -13,5 +14,11 @@ ts_devserver(
 benchmark_test(
     name = "perf",
     server = ":devserver",
-    deps = ["//modules/benchmarks/src/largetable:perf_lib"],
+    deps = ["//modules/benchmarks/src/largetable:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = ["//modules/benchmarks/src/largetable:e2e_tests_lib"],
 )

modules/benchmarks/src/largetable/largetable.spec.ts

@@ -0,0 +1,29 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {$} from 'protractor';
+
+import {openBrowser, verifyNoBrowserErrors} from '../../../e2e_util/e2e_util';
+
+describe('largetable benchmark', () => {
+  afterEach(verifyNoBrowserErrors);
+
+  it(`should render the table`, () => {
+    openBrowser({
+      url: '',
+      ignoreBrowserSynchronization: true,
+      params: [{name: 'cols', value: 5}, {name: 'rows', value: 5}],
+    });
+    $('#createDom').click();
+    expect($('#root').getText()).toContain('0/0');
+    $('#createDom').click();
+    expect($('#root').getText()).toContain('A/A');
+    $('#destroyDom').click();
+    expect($('#root').getText() as any).toEqual('');
+  });
+});

modules/benchmarks/src/largetable/largetable_perf.spec.ts

@@ -7,8 +7,6 @@
  */
 
 import {$} from 'protractor';
-
-import {openBrowser} from '../../../e2e_util/e2e_util';
 import {runBenchmark, verifyNoBrowserErrors} from '../../../e2e_util/perf_util';
 
 interface Worker {
@@ -48,20 +46,6 @@ describe('largetable benchmark perf', () => {
 
   afterEach(verifyNoBrowserErrors);
 
-  it(`should render the table for ${testPackageName}`, () => {
-    openBrowser({
-      url: '',
-      ignoreBrowserSynchronization: true,
-      params: [{name: 'cols', value: 5}, {name: 'rows', value: 5}],
-    });
-    $('#createDom').click();
-    expect($('#root').getText()).toContain('0/0');
-    $('#createDom').click();
-    expect($('#root').getText()).toContain('A/A');
-    $('#destroyDom').click();
-    expect($('#root').getText() as any).toEqual('');
-  });
-
   [CreateOnlyWorker, CreateAndDestroyWorker, UpdateWorker].forEach((worker) => {
     describe(worker.id, () => {
       it(`should run benchmark for ${testPackageName}`, done => {

modules/benchmarks/src/largetable/ng2/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ng_module", "ng_rollup_bundle", "ts_devserver")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -40,5 +41,11 @@ ts_devserver(
 benchmark_test(
     name = "perf",
     server = ":prodserver",
-    deps = ["//modules/benchmarks/src/largetable:perf_lib"],
+    deps = ["//modules/benchmarks/src/largetable:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":prodserver",
+    deps = ["//modules/benchmarks/src/largetable:e2e_tests_lib"],
 )

modules/benchmarks/src/largetable/ng2_switch/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ng_module", "ts_devserver")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -36,5 +37,11 @@ ts_devserver(
 benchmark_test(
     name = "perf",
     server = ":devserver",
-    deps = ["//modules/benchmarks/src/largetable:perf_lib"],
+    deps = ["//modules/benchmarks/src/largetable:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = ["//modules/benchmarks/src/largetable:e2e_tests_lib"],
 )

modules/benchmarks/src/largetable/render3/BUILD.bazel

@@ -2,6 +2,7 @@ package(default_visibility = ["//visibility:public"])
 
 load("//tools:defaults.bzl", "ng_module", "ng_rollup_bundle", "ts_devserver")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 ng_module(
     name = "largetable_lib",
@@ -43,5 +44,12 @@ benchmark_test(
     name = "perf",
     server = ":devserver",
     tags = ["ivy-only"],
-    deps = ["//modules/benchmarks/src/largetable:perf_lib"],
+    deps = ["//modules/benchmarks/src/largetable:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    tags = ["ivy-only"],
+    deps = ["//modules/benchmarks/src/largetable:e2e_tests_lib"],
 )

modules/benchmarks/src/tree/BUILD.bazel

@@ -11,7 +11,7 @@ ts_library(
 ts_library(
     name = "test_utils_lib",
     testonly = 1,
-    srcs = ["tree_perf_test_utils.ts"],
+    srcs = ["test_utils.ts"],
     deps = [
         "//modules/e2e_util",
         "@npm//protractor",
@@ -19,7 +19,7 @@ ts_library(
 )
 
 ts_library(
-    name = "perf_lib",
+    name = "perf_tests_lib",
     testonly = 1,
     srcs = ["tree_perf.spec.ts"],
     deps = [
@@ -29,7 +29,17 @@ ts_library(
 )
 
 ts_library(
-    name = "perf_detect_changes_lib",
+    name = "e2e_tests_lib",
+    testonly = 1,
+    srcs = ["tree.spec.ts"],
+    deps = [
+        ":test_utils_lib",
+        "@npm//protractor",
+    ],
+)
+
+ts_library(
+    name = "detect_changes_perf_tests_lib",
     testonly = 1,
     srcs = ["tree_perf_detect_changes.spec.ts"],
     deps = [
@@ -37,3 +47,13 @@ ts_library(
         "@npm//protractor",
     ],
 )
+
+ts_library(
+    name = "detect_changes_e2e_tests_lib",
+    testonly = 1,
+    srcs = ["tree_detect_changes.spec.ts"],
+    deps = [
+        ":test_utils_lib",
+        "@npm//protractor",
+    ],
+)

modules/benchmarks/src/tree/baseline/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ts_devserver", "ts_library")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -24,5 +25,11 @@ ts_devserver(
 benchmark_test(
     name = "perf",
     server = ":devserver",
-    deps = ["//modules/benchmarks/src/tree:perf_lib"],
+    deps = ["//modules/benchmarks/src/tree:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = ["//modules/benchmarks/src/tree:e2e_tests_lib"],
 )

modules/benchmarks/src/tree/incremental_dom/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ts_devserver", "ts_library")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -28,5 +29,11 @@ ts_devserver(
 benchmark_test(
     name = "perf",
     server = ":devserver",
-    deps = ["//modules/benchmarks/src/tree:perf_lib"],
+    deps = ["//modules/benchmarks/src/tree:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = ["//modules/benchmarks/src/tree:e2e_tests_lib"],
 )

modules/benchmarks/src/tree/iv/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ts_devserver")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -14,7 +15,16 @@ benchmark_test(
     name = "perf",
     server = ":devserver",
     deps = [
-        "//modules/benchmarks/src/tree:perf_detect_changes_lib",
-        "//modules/benchmarks/src/tree:perf_lib",
+        "//modules/benchmarks/src/tree:detect_changes_perf_tests_lib",
+        "//modules/benchmarks/src/tree:perf_tests_lib",
+    ],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = [
+        "//modules/benchmarks/src/tree:detect_changes_e2e_tests_lib",
+        "//modules/benchmarks/src/tree:e2e_tests_lib",
     ],
 )

modules/benchmarks/src/tree/ng1/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ts_devserver", "ts_library")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -26,7 +27,16 @@ benchmark_test(
     name = "perf",
     server = ":devserver",
     deps = [
-        "//modules/benchmarks/src/tree:perf_detect_changes_lib",
-        "//modules/benchmarks/src/tree:perf_lib",
+        "//modules/benchmarks/src/tree:detect_changes_perf_tests_lib",
+        "//modules/benchmarks/src/tree:perf_tests_lib",
+    ],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = [
+        "//modules/benchmarks/src/tree:detect_changes_e2e_tests_lib",
+        "//modules/benchmarks/src/tree:e2e_tests_lib",
     ],
 )

modules/benchmarks/src/tree/ng2/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ng_module", "ng_rollup_bundle", "ts_devserver")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -41,7 +42,16 @@ benchmark_test(
     name = "perf",
     server = ":prodserver",
     deps = [
-        "//modules/benchmarks/src/tree:perf_detect_changes_lib",
-        "//modules/benchmarks/src/tree:perf_lib",
+        "//modules/benchmarks/src/tree:detect_changes_perf_tests_lib",
+        "//modules/benchmarks/src/tree:perf_tests_lib",
+    ],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":prodserver",
+    deps = [
+        "//modules/benchmarks/src/tree:detect_changes_e2e_tests_lib",
+        "//modules/benchmarks/src/tree:e2e_tests_lib",
     ],
 )

modules/benchmarks/src/tree/ng2_next/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ts_devserver", "ts_library")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -32,7 +33,16 @@ benchmark_test(
     name = "perf",
     server = ":devserver",
     deps = [
-        "//modules/benchmarks/src/tree:perf_detect_changes_lib",
-        "//modules/benchmarks/src/tree:perf_lib",
+        "//modules/benchmarks/src/tree:detect_changes_perf_tests_lib",
+        "//modules/benchmarks/src/tree:perf_tests_lib",
+    ],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = [
+        "//modules/benchmarks/src/tree:detect_changes_e2e_tests_lib",
+        "//modules/benchmarks/src/tree:e2e_tests_lib",
     ],
 )

modules/benchmarks/src/tree/ng2_static/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ts_devserver", "ts_library")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -35,5 +36,11 @@ ts_devserver(
 benchmark_test(
     name = "perf",
     server = ":devserver",
-    deps = ["//modules/benchmarks/src/tree:perf_lib"],
+    deps = ["//modules/benchmarks/src/tree:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = ["//modules/benchmarks/src/tree:e2e_tests_lib"],
 )

modules/benchmarks/src/tree/ng2_switch/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ng_module", "ts_devserver")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -36,5 +37,11 @@ ts_devserver(
 benchmark_test(
     name = "perf",
     server = ":devserver",
-    deps = ["//modules/benchmarks/src/tree:perf_lib"],
+    deps = ["//modules/benchmarks/src/tree:perf_tests_lib"],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = ["//modules/benchmarks/src/tree:e2e_tests_lib"],
 )

modules/benchmarks/src/tree/render3/BUILD.bazel

@@ -2,6 +2,7 @@ package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
 load("//tools:defaults.bzl", "ng_module", "ng_rollup_bundle", "ts_devserver")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 ng_module(
     name = "tree_lib",
@@ -41,7 +42,17 @@ benchmark_test(
     server = ":devserver",
     tags = ["ivy-only"],
     deps = [
-        "//modules/benchmarks/src/tree:perf_detect_changes_lib",
-        "//modules/benchmarks/src/tree:perf_lib",
+        "//modules/benchmarks/src/tree:detect_changes_perf_tests_lib",
+        "//modules/benchmarks/src/tree:perf_tests_lib",
+    ],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    tags = ["ivy-only"],
+    deps = [
+        "//modules/benchmarks/src/tree:detect_changes_e2e_tests_lib",
+        "//modules/benchmarks/src/tree:e2e_tests_lib",
     ],
 )

modules/benchmarks/src/tree/render3_function/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//tools:defaults.bzl", "ts_devserver", "ts_library")
 load("//modules/benchmarks:benchmark_test.bzl", "benchmark_test")
+load("//modules/benchmarks:e2e_test.bzl", "e2e_test")
 
 package(default_visibility = ["//modules/benchmarks:__subpackages__"])
 
@@ -30,7 +31,16 @@ benchmark_test(
     name = "perf",
     server = ":devserver",
     deps = [
-        "//modules/benchmarks/src/tree:perf_detect_changes_lib",
-        "//modules/benchmarks/src/tree:perf_lib",
+        "//modules/benchmarks/src/tree:detect_changes_perf_tests_lib",
+        "//modules/benchmarks/src/tree:perf_tests_lib",
+    ],
+)
+
+e2e_test(
+    name = "e2e",
+    server = ":devserver",
+    deps = [
+        "//modules/benchmarks/src/tree:detect_changes_e2e_tests_lib",
+        "//modules/benchmarks/src/tree:e2e_tests_lib",
     ],
 )

modules/benchmarks/src/tree/tree.spec.ts

@@ -0,0 +1,28 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {$} from 'protractor';
+
+import {openTreeBenchmark} from './test_utils';
+
+describe('tree benchmark', () => {
+  it('should work for createDestroy', () => {
+    openTreeBenchmark();
+    $('#createDom').click();
+    expect($('#root').getText()).toContain('1');
+    $('#destroyDom').click();
+    expect($('#root').getText() as any).toEqual('');
+  });
+
+  it('should work for update', () => {
+    openTreeBenchmark();
+    $('#createDom').click();
+    $('#createDom').click();
+    expect($('#root').getText()).toContain('A');
+  });
+});

modules/benchmarks/src/tree/tree_detect_changes.spec.ts

@@ -0,0 +1,19 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {$} from 'protractor';
+
+import {openTreeBenchmark} from './test_utils';
+
+describe('tree benchmark detect changes', () => {
+  it('should work for detectChanges', () => {
+    openTreeBenchmark();
+    $('#detectChanges').click();
+    expect($('#numberOfChecks').getText()).toContain('10');
+  });
+});

modules/benchmarks/src/tree/tree_perf.spec.ts

@@ -7,27 +7,9 @@
  */
 
 import {$} from 'protractor';
-import {openTreeBenchmark, runTreeBenchmark} from './tree_perf_test_utils';
-
-describe('benchmark render', () => {
-  it('should work for createDestroy', () => {
-    openTreeBenchmark();
-    $('#createDom').click();
-    expect($('#root').getText()).toContain('0');
-    $('#destroyDom').click();
-    expect($('#root').getText() as any).toEqual('');
-  });
-
-  it('should work for update', () => {
-    openTreeBenchmark();
-    $('#createDom').click();
-    $('#createDom').click();
-    expect($('#root').getText()).toContain('A');
-  });
-});
-
-describe('benchmarks', () => {
+import {runTreeBenchmark} from './test_utils';
 
+describe('tree benchmark perf', () => {
   it('should work for createOnly', done => {
     runTreeBenchmark({
       // This cannot be called "createOnly" because the actual destroy benchmark

modules/benchmarks/src/tree/tree_perf_detect_changes.spec.ts

@@ -7,17 +7,9 @@
  */
 
 import {$} from 'protractor';
-import {openTreeBenchmark, runTreeBenchmark} from './tree_perf_test_utils';
+import {runTreeBenchmark} from './test_utils';
 
-describe('benchmark render', () => {
-  it('should work for detectChanges', () => {
-    openTreeBenchmark();
-    $('#detectChanges').click();
-    expect($('#numberOfChecks').getText()).toContain('10');
-  });
-});
-
-describe('benchmarks', () => {
+describe('tree benchmark detect changes perf', () => {
   it('should work for detectChanges', async() => {
     await runTreeBenchmark({
       id: 'detectChanges',

modules/e2e_util/e2e_util.ts

@@ -12,7 +12,7 @@ import {browser} from 'protractor';
 const yargs = require('yargs');
 import * as webdriver from 'selenium-webdriver';
 
-let cmdArgs: {'bundles': boolean};
+let cmdArgs: {'bundles': boolean}|undefined;
 
 declare var expect: any;
 
@@ -39,7 +39,7 @@ export function openBrowser(config: {
     browser.ignoreSynchronization = true;
   }
   let params = config.params || [];
-  if (!params.some((param) => param.name === 'bundles')) {
+  if (cmdArgs !== undefined && !params.some((param) => param.name === 'bundles')) {
     params = params.concat([{name: 'bundles', value: cmdArgs.bundles}]);
   }