diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index c15e4825..205edd40 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -350,11 +350,11 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v2
         with:
-          name: test-videos
-          path: ./test/e2e/videos
+          name: failed-test-videos
+          path: ./test/test-results
 
       - name: Remove release packages and test artifacts
-        run: rm -rf ./release-packages ./test/e2e/videos
+        run: rm -rf ./release-packages ./test/test-results
 
   docker-amd64:
     runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
index 45da9b42..736e30c5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,5 +16,5 @@ node-*
 .home
 coverage
 **/.DS_Store
-test/e2e/videos
-test/e2e/screenshots
+# Failed e2e test videos are saved here
+test/test-results
diff --git a/test/config.ts b/test/config.ts
index d0562665..e5ba91cd 100644
--- a/test/config.ts
+++ b/test/config.ts
@@ -51,7 +51,11 @@ globalSetup(async () => {
 const config: Config = {
   testDir: path.join(__dirname, "e2e"), // Search for tests in this directory.
   timeout: 30000, // Each test is given 30 seconds.
-  retries: 2, // Retry failing tests 2 times
+  retries: 3, // Retry failing tests 2 times
+}
+
+if (process.env.CI) {
+  config.retries = 2
 }
 
 setConfig(config)
diff --git a/test/e2e/browser.test.ts b/test/e2e/browser.test.ts
index fe1ab0e6..c5537ba0 100644
--- a/test/e2e/browser.test.ts
+++ b/test/e2e/browser.test.ts
@@ -1,22 +1,15 @@
 import { test, expect } from "@playwright/test"
+import { CODE_SERVER_ADDRESS } from "../utils/constants"
 
-test("should display correct browser based on userAgent", async ({ page, browserName }) => {
+// This is a "gut-check" test to make sure playwright is working as expected
+test("browser should display correct userAgent", async ({ page, browserName }) => {
   const displayNames = {
     chromium: "Chrome",
     firefox: "Firefox",
     webkit: "Safari",
   }
+  await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
   const userAgent = await page.evaluate("navigator.userAgent")
 
-  if (browserName === "chromium") {
-    expect(userAgent).toContain(displayNames[browserName])
-  }
-
-  if (browserName === "firefox") {
-    expect(userAgent).toContain(displayNames[browserName])
-  }
-
-  if (browserName === "webkit") {
-    expect(userAgent).toContain(displayNames[browserName])
-  }
+  expect(userAgent).toContain(displayNames[browserName])
 })