test: support BROWSER param, run tests in all browsers (#76)

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        browser: [chromium]
+        browser: [chromium, firefox, webkit]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2

playwright/pom.xml

@@ -26,7 +26,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>3.0.0-M4</version>
+        <version>3.0.0-M5</version>
       </plugin>
     </plugins>
   </build>

playwright/src/test/java/com/microsoft/playwright/TestAccessibility.java

@@ -18,6 +18,8 @@ package com.microsoft.playwright;
 
 import com.google.gson.*;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
+import org.junit.jupiter.api.condition.EnabledIf;
 
 import java.lang.reflect.Type;
 
@@ -145,7 +147,7 @@ public class TestAccessibility extends TestBase {
     // autofocus happens after a delay in chrome these days
     page.waitForFunction("() => document.activeElement.hasAttribute('autofocus')");
 
-    String golden = isFirefox ? "{\n" +
+    String golden = isFirefox() ? "{\n" +
       "  role: 'document',\n" +
       "  name: 'Accessibility Test',\n" +
       "  children: [\n" +
@@ -158,7 +160,7 @@ public class TestAccessibility extends TestBase {
       "    {role: 'textbox', name: '', valueString: 'and a value'}, // firefox doesn't use aria-placeholder for the name\n" +
       "    {role: 'textbox', name: '', valueString: 'and a value', description: 'This is a description!'} // and here\n" +
       "  ]\n" +
-      "}" : isChromium ? "{\n" +
+      "}" : isChromium() ? "{\n" +
       "  role: 'WebArea',\n" +
       "  name: 'Accessibility Test',\n" +
       "  children: [\n" +
@@ -193,7 +195,7 @@ public class TestAccessibility extends TestBase {
     page.setContent("<div>Hello World</div>");
     AccessibilityNode snapshot = page.accessibility().snapshot();
     AccessibilityNode node = snapshot.children().get(0);
-    assertEquals(isFirefox ? "text leaf" : "text", node.role());
+    assertEquals(isFirefox() ? "text leaf" : "text", node.role());
     assertEquals("Hello World", node.name());
   }
 
@@ -239,7 +241,7 @@ public class TestAccessibility extends TestBase {
       "  <div role='tab'>Tab2</div>\n" +
       "</div>");
     String golden = "{\n" +
-      "  role: '" + (isFirefox ? "document" : "WebArea") + "',\n" +
+      "  role: '" + (isFirefox() ? "document" : "WebArea") + "',\n" +
       "  name: '',\n" +
       "  children: [{\n" +
       "    role: 'tab',\n" +
@@ -254,12 +256,12 @@ public class TestAccessibility extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isWebKit", disabledReason="skip")
   void richTextEditableFieldsShouldHaveChildren() {
-// TODO:   test.skip(browserName === "webkit", "WebKit rich text accessibility is iffy");
     page.setContent("<div contenteditable='true'>\n" +
       "  Edit this image: <img src='fakeimage.png' alt='my fake image'>\n" +
       "</div>");
-    String golden = isFirefox ? "{\n" +
+    String golden = isFirefox() ? "{\n" +
       "  role: 'section',\n" +
       "  name: '',\n" +
       "  children: [{\n" +
@@ -286,12 +288,12 @@ public class TestAccessibility extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isWebKit", disabledReason="skip")
   void richTextEditableFieldsWithRoleShouldHaveChildren() {
-// TODO:   test.skip(browserName === "webkit", "WebKit rich text accessibility is iffy");
     page.setContent("<div contenteditable='true' role=\"textbox\">\n" +
       "  Edit this image: <img src='fakeimage.png' alt='my fake image'>\n" +
       "</div>");
-    String golden = isFirefox ? "{\n" +
+    String golden = isFirefox() ? "{\n" +
       "  role: 'textbox',\n" +
       "  name: '',\n" +
       "  valueString: 'Edit this image: my fake image',\n" +
@@ -315,9 +317,8 @@ public class TestAccessibility extends TestBase {
     assertNodeEquals(golden, snapshot.children().get(0));
   }
 
-  // TODO:   suite.skip(browserName === "firefox", "Firefox does not support contenteditable='plaintext-only'");
-// TODO:   suite.skip(browserName === "webkit", "WebKit rich text accessibility is iffy");
   @Test
+  @EnabledIf(value="com.microsoft.playwright.TestBase#isChromium", disabledReason="skip")
   void plainTextFieldWithRoleShouldNotHaveChildren() {
     page.setContent("<div contenteditable='plaintext-only' role='textbox'>Edit this image:<img src='fakeimage.png' alt='my fake image'></div>");
     AccessibilityNode snapshot = page.accessibility().snapshot();
@@ -329,6 +330,7 @@ public class TestAccessibility extends TestBase {
   }
 
   @Test
+  @EnabledIf(value="com.microsoft.playwright.TestBase#isChromium", disabledReason="skip")
   void plainTextFieldWithoutRoleShouldNotHaveContent() {
     page.setContent("<div contenteditable='plaintext-only'>Edit this image:<img src='fakeimage.png' alt='my fake image'></div>");
     AccessibilityNode snapshot = page.accessibility().snapshot();
@@ -339,6 +341,7 @@ public class TestAccessibility extends TestBase {
   }
 
   @Test
+  @EnabledIf(value="com.microsoft.playwright.TestBase#isChromium", disabledReason="skip")
   void plainTextFieldWithTabindexAndWithoutRoleShouldNotHaveContent() {
     page.setContent("<div contenteditable='plaintext-only' tabIndex=0>Edit this image:<img src='fakeimage.png' alt='my fake image'></div>");
     AccessibilityNode snapshot = page.accessibility().snapshot();
@@ -354,18 +357,18 @@ public class TestAccessibility extends TestBase {
       "this is the inner content\n" +
       "<img alt='yo' src='fakeimg.png'>\n" +
       "</div>");
-    String golden = isFirefox ? "{\n" +
+    String golden = isFirefox() ? "{\n" +
       "  role: 'textbox',\n" +
       "  name: 'my favorite textbox',\n" +
       "  valueString: 'this is the inner content yo'\n" +
-      "}" : isChromium ? "{\n" +
+      "}" : isChromium() ? "{\n" +
       "  role: 'textbox',\n" +
       "  name: 'my favorite textbox',\n" +
       "  valueString: 'this is the inner content '\n" +
       "}" : "{\n" +
       "  role: 'textbox',\n" +
       "  name: 'my favorite textbox',\n" +
-      "  valueString: 'this is the inner content  ',\n" +
+      "  valueString: 'this is the inner content '\n" +
       "}";
     AccessibilityNode snapshot = page.accessibility().snapshot();
     assertNodeEquals(golden, snapshot.children().get(0));
@@ -392,7 +395,7 @@ public class TestAccessibility extends TestBase {
       "this is the inner content\n" +
       "<img alt='yo' src='fakeimg.png'>\n" +
       "</div>");
-    String golden = isFirefox ? "{\n" +
+    String golden = isFirefox() ? "{\n" +
       "  role: 'checkbox',\n" +
       "  name: 'this is the inner content yo',\n" +
       "  checked: 'checked'\n" +
@@ -444,7 +447,7 @@ public class TestAccessibility extends TestBase {
       "  [ { role: 'menuitem', name: 'First Item' },\n" +
       "    { role: 'menuitem', name: 'Second Item' },\n" +
       "    { role: 'menuitem', name: 'Third Item' } ]\n" +
-      (isWebKit ? ", orientation: 'vertical'" : "") +
+      (isWebKit() ? ", orientation: 'vertical'" : "") +
       "  }", page.accessibility().snapshot(new Accessibility.SnapshotOptions().withRoot(menu)));
   }
 

playwright/src/test/java/com/microsoft/playwright/TestBase.java

@@ -29,24 +29,55 @@ public class TestBase {
   static BrowserType browserType;
   static Playwright playwright;
   static Browser browser;
-  static boolean isChromium;
-  static boolean isWebKit;
-  static boolean isFirefox;
   static boolean isMac = Utils.getOS() == Utils.OS.MAC;
   static boolean isWindows = Utils.getOS() == Utils.OS.WINDOWS;
   static boolean headful;
   Page page;
   BrowserContext context;
 
+  static boolean isHeadful() {
+    return headful;
+  }
+
+  static boolean isChromium() {
+    return "chromium".equals(browserType.name());
+  }
+
+  static boolean isWebKit() {
+    return "webkit".equals(browserType.name());
+  }
+
+  static boolean isFirefox() {
+    return "firefox".equals(browserType.name());
+  }
+
   @BeforeAll
   static void launchBrowser() {
     playwright = Playwright.create();
+
+
+    String browserName = System.getenv("BROWSER");
+    if (browserName == null) {
+      browserName = "chromium";
+    }
+    switch (browserName) {
+      case "webkit":
+        browserType = playwright.webkit();
+        break;
+      case "firefox":
+        browserType = playwright.firefox();
+        break;
+      case "chromium":
+        browserType = playwright.chromium();
+        break;
+      default:
+        throw new IllegalArgumentException("Unknown browser: " + browserName);
+    }
+    String headfulEnv = System.getenv("HEADFUL");
+    headful = headfulEnv != null && !"0".equals(headfulEnv) && !"false".equals(headfulEnv);
     BrowserType.LaunchOptions options = new BrowserType.LaunchOptions();
-    browserType = playwright.chromium();
+    options.headless = !headful;
     browser = browserType.launch(options);
-    isChromium = true;
-    isWebKit = false;
-    headful = false;
   }
 
   @AfterAll

playwright/src/test/java/com/microsoft/playwright/TestBrowserContextAddCookies.java

@@ -17,8 +17,6 @@
 package com.microsoft.playwright;
 
 import com.google.gson.Gson;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonParser;
 import org.junit.jupiter.api.Test;
 
 import java.time.Instant;
@@ -352,7 +350,7 @@ public class TestBrowserContextAddCookies extends TestBase {
       "}", server.CROSS_PROCESS_PREFIX + "/grid.html");
     page.frames().get(1).evaluate("document.cookie = 'username=John Doe'");
     page.waitForTimeout(2000);
-    boolean allowsThirdParty = isChromium || isFirefox;
+    boolean allowsThirdParty = isChromium() || isFirefox();
     List<BrowserContext.Cookie> cookies = context.cookies(server.CROSS_PROCESS_PREFIX + "/grid.html");
     if (allowsThirdParty) {
       assertJsonEquals("[{\n" +

playwright/src/test/java/com/microsoft/playwright/TestBrowserContextCookies.java

@@ -17,11 +17,13 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
 
 import java.util.Comparator;
 import java.util.List;
 
 import static com.microsoft.playwright.Utils.assertJsonEquals;
+import static com.microsoft.playwright.Utils.getOS;
 import static java.util.Arrays.asList;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -84,9 +86,13 @@ public class TestBrowserContextCookies extends TestBase {
     assertTrue(cookies.get(0).httpOnly());
   }
 
+  static boolean isWebKitWindows() {
+    return isWebKit() && getOS() == Utils.OS.WINDOWS;
+  }
+
   @Test
+  @DisabledIf(value="isWebKitWindows", disabledReason="fail")
   void shouldProperlyReportStrictSameSiteCookie() {
-// TODO:   test.fail(browserName === "webkit" && platform === "win32");
     server.setRoute("/empty.html", exchange -> {
       exchange.getResponseHeaders().add("Set-Cookie", "name=value;SameSite=Strict");
       exchange.sendResponseHeaders(200, 0);
@@ -99,8 +105,8 @@ public class TestBrowserContextCookies extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="isWebKitWindows", disabledReason="fail")
   void shouldProperlyReportLaxSameSiteCookie() {
-// TODO:   test.fail(browserName === "webkit" && platform === "win32");
     server.setRoute("/empty.html", exchange -> {
       exchange.getResponseHeaders().add("Set-Cookie", "name=value;SameSite=Lax");
       exchange.sendResponseHeaders(200, 0);

playwright/src/test/java/com/microsoft/playwright/TestBrowserContextCredentials.java

@@ -17,14 +17,21 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
 
+import static com.microsoft.playwright.Utils.getOS;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestBrowserContextCredentials extends TestBase {
+
+  static boolean isChromiumHeadful() {
+    return isChromium() && isHeadful();
+  }
+
   @Test
+  @DisabledIf(value="isChromiumHeadful", disabledReason="fail")
   void shouldFailWithoutCredentials() {
-// TODO:   test.fail(browserName === "chromium" && headful);
     server.setAuth("/empty.html", "user", "pass");
     BrowserContext context = browser.newContext();
     Page page = context.newPage();

playwright/src/test/java/com/microsoft/playwright/TestClick.java

@@ -17,6 +17,7 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -314,8 +315,8 @@ public class TestClick extends TestBase {
     page.click("button", new Page.ClickOptions().withPosition(20, 10));
     assertEquals(page.evaluate("result"), "Clicked");
     // Safari reports border-relative offsetX/offsetY.
-    assertEquals(isWebKit ? 20 + 8 : 20, page.evaluate("offsetX"));
+    assertEquals(isWebKit() ? 20 + 8 : 20, page.evaluate("offsetX"));
-    assertEquals(isWebKit ? 10 + 8 : 10, page.evaluate("offsetY"));
+    assertEquals(isWebKit() ? 10 + 8 : 10, page.evaluate("offsetY"));
   }
 
   @Test
@@ -326,8 +327,8 @@ public class TestClick extends TestBase {
     page.click("button", new Page.ClickOptions().withPosition(20, 10));
     assertEquals("Clicked", page.evaluate("result"));
     // Safari reports border-relative offsetX/offsetY.
-    assertEquals(isWebKit ? 12 * 2 + 20 : 20, page.evaluate("offsetX"));
+    assertEquals(isWebKit() ? 12 * 2 + 20 : 20, page.evaluate("offsetX"));
-    assertEquals(isWebKit ? 12 * 2 + 10 : 10, page.evaluate("offsetY"));
+    assertEquals(isWebKit() ? 12 * 2 + 10 : 10, page.evaluate("offsetY"));
   }
 
   @Test
@@ -338,8 +339,8 @@ public class TestClick extends TestBase {
     page.click("button", new Page.ClickOptions().withPosition(1900, 1910));
     assertEquals("Clicked", page.evaluate("() => window['result']"));
     // Safari reports border-relative offsetX/offsetY.
-    assertEquals(isWebKit ? 1900 + 8 : 1900, page.evaluate("offsetX"));
+    assertEquals(isWebKit() ? 1900 + 8 : 1900, page.evaluate("offsetX"));
-    assertEquals(isWebKit ? 1910 + 8 : 1910, page.evaluate("offsetY"));
+    assertEquals(isWebKit() ? 1910 + 8 : 1910, page.evaluate("offsetY"));
   }
 
   @Test
@@ -359,13 +360,13 @@ public class TestClick extends TestBase {
     page.click("button", new Page.ClickOptions().withPosition(1900, 1910));
     assertEquals("Clicked", page.evaluate("() => window['result']"));
     // Safari reports border-relative offsetX/offsetY.
-    assertEquals(isWebKit ? 1900 + 8 : 1900, page.evaluate("offsetX"));
+    assertEquals(isWebKit() ? 1900 + 8 : 1900, page.evaluate("offsetX"));
-    assertEquals(isWebKit ? 1910 + 8 : 1910, page.evaluate("offsetY"));
+    assertEquals(isWebKit() ? 1910 + 8 : 1910, page.evaluate("offsetY"));
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isFirefox", disabledReason="skip")
   void shouldClickTheButtonWithOffsetWithPageScale() {
-    // TODO:    test.skip(browserName === "firefox");
     BrowserContext context = browser.newContext(new Browser.NewContextOptions()
       .withViewport(400, 400)
       .withIsMobile(true));
@@ -380,11 +381,11 @@ public class TestClick extends TestBase {
     // 20;10 + 8px of border in each direction
     int expectedX = 28;
     int expectedY = 18;
-    if (isWebKit) {
+    if (isWebKit()) {
       // WebKit rounds up during css -> dip -> css conversion.
       expectedX = 29;
       expectedY = 19;
-    } else if (isChromium && !headful) {
+    } else if (isChromium() && !headful) {
       // Headless Chromium rounds down during css -> dip -> css conversion.
       expectedX = 27;
       expectedY = 18;

playwright/src/test/java/com/microsoft/playwright/TestDefaultBrowserContext2.java

@@ -2,6 +2,7 @@ package com.microsoft.playwright;
 
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -53,8 +54,8 @@ public class TestDefaultBrowserContext2 extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isFirefox", disabledReason="skip")
   void shouldWorkInPersistentContext() {
-// TODO:   test.skip(browserName === "firefox");
     // Firefox does not support mobile.
     Page page = launchPersistent(new BrowserType.LaunchPersistentContextOptions()
       .withViewport(320, 480).withIsMobile(true));
@@ -185,8 +186,8 @@ public class TestDefaultBrowserContext2 extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isFirefox", disabledReason="skip")
   void shouldThrowIfPageArgumentIsPassed() throws IOException {
-// TODO:   test.skip(browserName === "firefox");
     BrowserType.LaunchPersistentContextOptions options = new BrowserType.LaunchPersistentContextOptions()
       .withArgs(asList(server.EMPTY_PAGE));
     Path userDataDir = Files.createTempDirectory("user-data-dir-");
@@ -199,15 +200,12 @@ public class TestDefaultBrowserContext2 extends TestBase {
   }
 
   void shouldHavePassedURLWhenLaunchingWithIgnoreDefaultArgsTrue() {
-//  test.skip(wire);
   }
 
   void shouldHandleTimeout() {
-//  test.skip(wire);
   }
 
   void shouldHandleException() {
-//  test.skip(wire);
   }
 
   @Test

playwright/src/test/java/com/microsoft/playwright/TestDialog.java

@@ -17,6 +17,8 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
+import org.junit.jupiter.api.condition.EnabledIf;
 
 import static com.microsoft.playwright.Dialog.Type.ALERT;
 import static com.microsoft.playwright.Dialog.Type.PROMPT;
@@ -81,9 +83,13 @@ public class TestDialog extends TestBase {
     assertEquals(false, result);
   }
 
+  static boolean isWebKitMac() {
+    return isWebKit() && Utils.getOS() == Utils.OS.MAC;
+  }
+
   @Test
+  @DisabledIf(value="isWebKitMac", disabledReason="fixme")
   void shouldBeAbleToCloseContextWithOpenAlert() {
-//    test.fixme(browserName === "webkit" && platform === "darwin");
     BrowserContext context = browser.newContext();
     Page page = context.newPage();
 //    const alertPromise = page.waitForEvent("dialog");

playwright/src/test/java/com/microsoft/playwright/TestDownload.java

@@ -18,6 +18,8 @@ package com.microsoft.playwright;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
+import org.junit.jupiter.api.condition.EnabledIf;
 
 import java.io.*;
 import java.nio.charset.StandardCharsets;
@@ -305,8 +307,8 @@ public class TestDownload extends TestBase {
   }
 
   @Test
+  @EnabledIf(value="com.microsoft.playwright.TestBase#isChromium", disabledReason="fixme")
   void shouldReportAltClickDownloads() throws IOException {
-// TODO:   test.fixme(browserName === "firefox" || browserName === "webkit");
     // Firefox does not download on alt-click by default.
     // Our WebKit embedder does not download on alt-click, although Safari does.
     server.setRoute("/download", exchange -> {
@@ -328,12 +330,22 @@ public class TestDownload extends TestBase {
     page.close();
   }
 
+
+  static boolean isChromiumHeadful() {
+    return isChromium() && isHeadful();
+  }
+
+  static boolean isChromiumHeadfulOrFirefox() {
+    // TODO: figure out why download is not received in Firefox.
+    return isChromiumHeadful() || isFirefox();
+  }
+
   @Test
+  @DisabledIf(value="isChromiumHeadfulOrFirefox", disabledReason="fixme")
   void shouldReportNewWindowDownloads() throws IOException {
-// TODO:    test.fixme(browserName === "chromium" && headful);
     // TODO: - the test fails in headful Chromium as the popup page gets closed along
     // with the session before download completed event arrives.
-    // - WebKit doesn"t close the popup page
+    // - WebKit doesn't close the popup page
     Page page = browser.newPage(new Browser.NewPageOptions().withAcceptDownloads(true));
     page.setContent("<a target=_blank href='" + server.PREFIX + "/download'>download</a>");
     Deferred<Event<Page.EventType>> downloadEvent = page.waitForEvent(DOWNLOAD);

playwright/src/test/java/com/microsoft/playwright/TestElementHandleBoundingBox.java

@@ -17,15 +17,21 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
 
 import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.*;
 
 public class TestElementHandleBoundingBox extends TestBase {
+
+  static boolean isFirefoxHeadful() {
+    return isFirefox() && isHeadful();
+  }
+
   @Test
+  @DisabledIf(value="isFirefoxHeadful", disabledReason="fail")
   void shouldWork() {
-    // TODO: test.fail(browserName === "firefox" && headful);
     page.setViewportSize(500, 500);
     page.navigate(server.PREFIX + "/grid.html");
     ElementHandle elementHandle = page.querySelector(".box:nth-of-type(13)");
@@ -89,8 +95,8 @@ public class TestElementHandleBoundingBox extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isFirefox", disabledReason="skip")
   void shouldWorkWithPageScale() {
-    // TODO: test.skip(browserName === "firefox");
     BrowserContext context = browser.newContext(new Browser.NewContextOptions()
       .withViewport(400, 400).withIsMobile(true));
     Page page = context.newPage();

playwright/src/test/java/com/microsoft/playwright/TestElementHandleWaitForElementState.java

@@ -17,6 +17,7 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
 
 import static com.microsoft.playwright.ElementHandle.ElementState.*;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -136,9 +137,12 @@ public class TestElementHandleWaitForElementState extends TestBase {
     promise.get();
   }
 
+  static boolean isFirefoxLinux() {
+    return isFirefox() && Utils.getOS() == Utils.OS.LINUX;
+  }
   @Test
+  @DisabledIf(value="isFirefoxLinux", disabledReason="fixme")
   void shouldWaitForStablePosition() {
-    // TODO: test.fixme(browserName === "firefox" && platform === "linux");
     page.navigate(server.PREFIX + "/input/button.html");
     ElementHandle button = page.querySelector("button");
     page.evalOnSelector("button", "button => {\n" +

playwright/src/test/java/com/microsoft/playwright/TestNetworkRequest.java

@@ -17,6 +17,7 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
 
 import java.io.OutputStreamWriter;
 import java.util.ArrayList;
@@ -25,8 +26,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static com.microsoft.playwright.Page.EventType.REQUEST;
 import static com.microsoft.playwright.Page.EventType.RESPONSE;
@@ -96,17 +95,17 @@ public class TestNetworkRequest extends TestBase {
   @Test
   void shouldReturnHeaders() {
     Response response = page.navigate(server.EMPTY_PAGE);
-    if (isChromium)
+    if (isChromium())
       assertTrue(response.request().headers().get("user-agent").contains("Chrome"));
-    else if (isFirefox)
+    else if (isFirefox())
       assertTrue(response.request().headers().get("user-agent").contains("Firefox"));
-    else if (isWebKit)
+    else if (isWebKit())
       assertTrue(response.request().headers().get("user-agent").contains("WebKit"));
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isWebKit", disabledReason="fail")
   void shouldGetTheSameHeadersAsTheServer() throws ExecutionException, InterruptedException {
-// TODO:    test.fail(browserName === "webkit", "Provisional headers differ from those in network stack");
     Future<Server.Request> serverRequest = server.waitForRequest("/empty.html");
     server.setRoute("/empty.html", exchange -> {
       exchange.sendResponseHeaders(200, 0);
@@ -122,8 +121,8 @@ public class TestNetworkRequest extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isWebKit", disabledReason="fail")
   void shouldGetTheSameHeadersAsTheServerCORP() throws ExecutionException, InterruptedException {
-// TODO:    test.fail(browserName === "webkit", "Provisional headers differ from those in network stack");
     page.navigate(server.PREFIX + "/empty.html");
     Future<Server.Request> serverRequest = server.waitForRequest("/something");
     server.setRoute("/something", exchange -> {

playwright/src/test/java/com/microsoft/playwright/TestNetworkResponse.java

@@ -23,7 +23,9 @@ import java.io.OutputStreamWriter;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.Semaphore;
 
 import static com.microsoft.playwright.Page.EventType.REQUESTFINISHED;
 import static com.microsoft.playwright.Page.EventType.RESPONSE;
@@ -75,8 +77,10 @@ public class TestNetworkResponse extends TestBase {
   }
 
   @Test
-  void shouldWaitUntilResponseCompletes() {
+  void shouldWaitUntilResponseCompletes() throws ExecutionException, InterruptedException {
     page.navigate(server.EMPTY_PAGE);
+    Semaphore responseWritten = new Semaphore(0);
+    Semaphore responseRead = new Semaphore(0);
     server.setRoute("/get", exchange -> {
       // In Firefox, |fetch| will be hanging until it receives |Content-Type| header
       // from server.
@@ -85,10 +89,15 @@ public class TestNetworkResponse extends TestBase {
       try (OutputStreamWriter writer = new OutputStreamWriter(exchange.getResponseBody())) {
         writer.write("hello ");
         writer.flush();
+        responseWritten.release();
+        responseRead.acquire();
         writer.write("wor");
         writer.flush();
         writer.write("ld!");
+      } catch (InterruptedException e) {
+        e.printStackTrace();
       }
+      responseWritten.release();
     });
     // Setup page to trap response.
     boolean[] requestFinished = {false};
@@ -97,13 +106,16 @@ public class TestNetworkResponse extends TestBase {
     });
     // send request and wait for server response
     Deferred<Event<Page.EventType>> responseEvent = page.waitForEvent(RESPONSE);
-    Future<Server.Request> request = server.waitForRequest("/get");
     page.evaluate("() => fetch('./get', { method: 'GET'})");
     assertNotNull(responseEvent.get());
+    responseWritten.acquire();
     Response pageResponse = (Response) responseEvent.get().data();
     assertEquals(200, pageResponse.status());
     assertEquals(false, requestFinished[0]);
+    responseRead.release();
+    responseWritten.acquire();
     assertEquals("hello world!", pageResponse.text());
+    assertEquals(true, requestFinished[0]);
   }
 
   void shouldReturnJson() {

playwright/src/test/java/com/microsoft/playwright/TestPageBasic.java

@@ -17,6 +17,7 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -64,9 +65,9 @@ public class TestPageBasic extends TestBase {
       Dialog dialog = (Dialog) event.data();
       assertEquals(Dialog.Type.BEFOREUNLOAD, dialog.type());
       assertEquals("", dialog.defaultValue());
-      if (isChromium) {
+      if (isChromium()) {
         assertEquals("", dialog.message());
-      } else if (isWebKit) {
+      } else if (isWebKit()) {
         assertEquals("Leave?", dialog.message());
       } else {
         assertEquals("This page is asking you to confirm that you want to leave - data you have entered may not be saved.", dialog.message());
@@ -241,7 +242,7 @@ public class TestPageBasic extends TestBase {
     // Second part in parenthesis is platform - ignore it.
 
     // Third part for Firefox is the last one and encodes engine and browser versions.
-    if (isFirefox) {
+    if (isFirefox()) {
       String[] engineAndBrowser = parts.get(2).split(" ");
       assertTrue(engineAndBrowser[0].startsWith("Gecko"));
       assertTrue(engineAndBrowser[1].startsWith("Firefox"));
@@ -253,7 +254,7 @@ public class TestPageBasic extends TestBase {
     // 5th part encodes real browser name and engine version.
     String[] engineAndBrowser = parts.get(4).split(" ");
     assertTrue(engineAndBrowser[1].startsWith("Safari"));
-    if (isChromium) {
+    if (isChromium()) {
       assertTrue(engineAndBrowser[0].contains("Chrome/"));
     } else {
       assertTrue(engineAndBrowser[0].startsWith("Version/"));
@@ -285,8 +286,8 @@ public class TestPageBasic extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isFirefox", disabledReason="fail")
   void frameFocusShouldWorkMultipleTimes() {
-    // TODO:    test.fail(browserName === "firefox");
     Page page1 = context.newPage();
     Page page2 = context.newPage();
     for (Page page : Arrays.asList(page1, page2)) {

playwright/src/test/java/com/microsoft/playwright/TestPageEvaluate.java

@@ -17,6 +17,8 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
+import org.junit.jupiter.api.condition.EnabledIf;
 
 import java.util.Map;
 
@@ -458,8 +460,8 @@ public class TestPageEvaluate extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isWebKit", disabledReason="fixme")
   void shouldNotThrowAnErrorWhenEvaluationDoesASynchronousNavigationAndReturnsAnObject() {
-// TODO: test.fixme(browserName === "webkit");
     // It is imporant to be on about:blank for sync reload.
     Object result = page.evaluate("() => {\n" +
       "  window.location.reload();\n" +

playwright/src/test/java/com/microsoft/playwright/TestPageEventNetwork.java

@@ -62,9 +62,9 @@ public class TestPageEventNetwork extends TestBase {
     assertTrue(failedRequests.get(0).url().contains("one-style.css"));
     assertNull(failedRequests.get(0).response());
     assertEquals("stylesheet", failedRequests.get(0).resourceType());
-    if (isChromium) {
+    if (isChromium()) {
       assertEquals("net::ERR_EMPTY_RESPONSE", failedRequests.get(0).failure().errorText());
-    } else if (isWebKit) {
+    } else if (isWebKit()) {
       if (isMac)
         assertEquals("The network connection was lost.", failedRequests.get(0).failure().errorText());
       else if (isWindows)

playwright/src/test/java/com/microsoft/playwright/TestPageFill.java

@@ -17,6 +17,8 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
+import org.junit.jupiter.api.condition.EnabledIf;
 
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -69,8 +71,8 @@ public class TestPageFill extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isWebKit", disabledReason="skip")
   void shouldThrowOnIncorrectDate() {
-    // TODO: test.skip(browserName === "webkit");
     page.setContent("<input type=date>");
     try {
       page.fill("input", "2020-13-05");
@@ -88,8 +90,8 @@ public class TestPageFill extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isWebKit", disabledReason="skip")
   void shouldThrowOnIncorrectTime() {
-    // TODO: test.skip(browserName === "webkit");
     page.setContent("<input type=time>");
     try {
       page.fill("input", "25:05");
@@ -107,8 +109,8 @@ public class TestPageFill extends TestBase {
   }
 
   @Test
+  @EnabledIf(value="com.microsoft.playwright.TestBase#isChromium", disabledReason="skip")
   void shouldThrowOnIncorrectDatetimeLocal() {
-    // TODO: test.skip(browserName === "webkit" || browserName === "firefox");
     page.setContent("<input type=datetime-local>");
     try {
       page.fill("input", "abc");

playwright/src/test/java/com/microsoft/playwright/TestPageRoute.java

@@ -17,13 +17,14 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
+import org.junit.jupiter.api.condition.EnabledIf;
 
 import java.io.OutputStreamWriter;
 import java.util.*;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.regex.Pattern;
 
@@ -119,16 +120,16 @@ public class TestPageRoute extends TestBase {
   // @see https://github.com/GoogleChrome/puppeteer/issues/4743
   @Test
   void shouldBeAbleToRemoveHeaders() throws ExecutionException, InterruptedException {
+    page.navigate(server.PREFIX + "/empty.html");
     page.route("**/*", route -> {
       Map<String, String> headers = new HashMap<>(route.request().headers());
-      headers.put("foo", "bar");
+      headers.remove("foo");
-      headers.remove("accept");
       route.continue_(new Route.ContinueOverrides().withHeaders(headers));
     });
 
-    Future<Server.Request> serverRequest = server.waitForRequest("/empty.html");
+    Future<Server.Request> serverRequest = server.waitForRequest("/title.html");
-    page.navigate(server.PREFIX + "/empty.html");
+    page.evaluate("url => fetch(url, { headers: {foo: 'bar'} })", server.PREFIX + "/title.html");
-    assertFalse(serverRequest.get().headers.containsKey("accept"));
+    assertFalse(serverRequest.get().headers.containsKey("foo"));
   }
 
   @Test
@@ -169,6 +170,7 @@ public class TestPageRoute extends TestBase {
 
   // @see https://github.com/GoogleChrome/puppeteer/issues/4337
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isWebKit", disabledReason="fixme")
   void shouldWorkWithRedirectInsideSyncXHR() {
     page.navigate(server.EMPTY_PAGE);
     server.setRedirect("/logo.png", "/pptr.png");
@@ -218,9 +220,9 @@ public class TestPageRoute extends TestBase {
     } catch (PlaywrightException e) {
     }
     assertNotNull(failedRequest[0]);
-    if (isWebKit)
+    if (isWebKit())
       assertEquals("Request intercepted", failedRequest[0].failure().errorText());
-    else if (isFirefox)
+    else if (isFirefox())
       assertEquals("NS_ERROR_OFFLINE", failedRequest[0].failure().errorText());
     else
       assertEquals("net::ERR_INTERNET_DISCONNECTED", failedRequest[0].failure().errorText());
@@ -242,9 +244,9 @@ public class TestPageRoute extends TestBase {
       page.navigate(server.EMPTY_PAGE);
       fail("did not throw");
     } catch (PlaywrightException e) {
-      if (isWebKit)
+      if (isWebKit())
         assertTrue(e.getMessage().contains("Request intercepted"));
-      else if (isFirefox)
+      else if (isFirefox())
         assertTrue(e.getMessage().contains("NS_ERROR_FAILURE"));
       else
         assertTrue(e.getMessage().contains("net::ERR_FAILED"));

playwright/src/test/java/com/microsoft/playwright/TestPdf.java

@@ -17,6 +17,8 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
+import org.junit.jupiter.api.condition.EnabledIf;
 
 import java.io.File;
 import java.io.IOException;
@@ -27,8 +29,9 @@ import static org.junit.jupiter.api.Assertions.*;
 
 public class TestPdf extends TestBase {
   @Test
+  @EnabledIf(value="com.microsoft.playwright.TestBase#isChromium", disabledReason="skip")
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isHeadful", disabledReason="skip")
   void shouldBeAbleToSaveFile() throws IOException {
-// TODO:   test.skip(headful || browserName !== "chromium", "Printing to pdf is currently only supported in headless chromium.");
     Path path = File.createTempFile("output", ".pdf").toPath();
     page.pdf(new Page.PdfOptions().withPath(path));
     long size = Files.size(path);
@@ -36,11 +39,11 @@ public class TestPdf extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="com.microsoft.playwright.TestBase#isChromium", disabledReason="skip")
   void shouldOnlyHavePdfInChromium() {
-// TODO:   test.skip(browserName === "chromium");
     try {
       page.pdf();
-      if (isChromium) {
+      if (isChromium()) {
         return;
       }
       fail("did not throw");

playwright/src/test/java/com/microsoft/playwright/TestRequestFulfill.java

@@ -17,6 +17,7 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIf;
 
 import java.io.File;
 import java.io.IOException;
@@ -55,9 +56,13 @@ public class TestRequestFulfill extends TestBase {
     assertEquals("Yo, page!", page.evaluate("document.body.textContent"));
   }
 
+  static boolean isFirefoxHeadful() {
+    return isFirefox() && isHeadful();
+  }
+
   @Test
+  @DisabledIf(value="isFirefoxHeadful", disabledReason="skip")
   void shouldAllowMockingBinaryResponses() {
-// TODO:    test.skip(browserName === "firefox" && headful, "// Firefox headful produces a different image.");
     page.route("**/*", route -> {
       byte[] imageBuffer;
       try {
@@ -81,8 +86,8 @@ public class TestRequestFulfill extends TestBase {
   }
 
   @Test
+  @DisabledIf(value="isFirefoxHeadful", disabledReason="skip")
   void shouldAllowMockingSvgWithCharset() {
-    // TODO: test.skip(browserName === "firefox" && headful, "// Firefox headful produces a different image.");
     // Firefox headful produces a different image.
     page.route("**/*", route -> {
       route.fulfill(new Route.FulfillResponse()

playwright/src/test/java/com/microsoft/playwright/TestWorkers.java

@@ -17,6 +17,7 @@
 package com.microsoft.playwright;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIf;
 
 import static com.microsoft.playwright.Page.EventType.*;
 import static com.microsoft.playwright.Utils.attachFrame;
@@ -120,8 +121,9 @@ public class TestWorkers extends TestBase {
     assertEquals(0, page.workers().size());
   }
 
+  @Test
+  @EnabledIf(value="com.microsoft.playwright.TestBase#isWebKit", disabledReason="fixme")
   void shouldAttributeNetworkActivityForWorkerInsideIframeToTheIframe() {
-    // TODO: test.fixme(browserName === "firefox" || browserName === "chromium");
     page.navigate(server.PREFIX + "/empty.html");
     Deferred<Event<Page.EventType>> workerEvent = page.waitForEvent(WORKER);
     Frame frame = attachFrame(page, "frame1", server.PREFIX + "/worker/worker.html");