Merge pull request #3133 from cdr/jsjoeio/migrate-to-playwright-test

refactor(testing): migrate to playwright-test from jest-playwright
This commit is contained in:
repo-ranger[bot] 2021-04-15 19:04:24 +00:00 committed by GitHub
commit 97fbbfaecc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1294 additions and 731 deletions

View File

@ -350,11 +350,11 @@ jobs:
if: always() if: always()
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v2
with: with:
name: test-videos name: failed-test-videos
path: ./test/e2e/videos path: ./test/test-results
- name: Remove release packages and test artifacts - name: Remove release packages and test artifacts
run: rm -rf ./release-packages ./test/e2e/videos run: rm -rf ./release-packages ./test/test-results
docker-amd64: docker-amd64:
runs-on: ubuntu-latest runs-on: ubuntu-latest

4
.gitignore vendored
View File

@ -16,5 +16,5 @@ node-*
.home .home
coverage coverage
**/.DS_Store **/.DS_Store
test/e2e/videos # Failed e2e test videos are saved here
test/e2e/screenshots test/test-results

View File

@ -3,19 +3,10 @@ set -euo pipefail
main() { main() {
cd "$(dirname "$0")/../.." cd "$(dirname "$0")/../.."
# We must keep jest in a sub-directory. See ../../test/package.json for more cd test
# information. We must also run it from the root otherwise coverage will not # We set these environment variables because they're used in the e2e tests
# include our source files. # they don't have to be these values, but these are the defaults
if [[ -z ${PASSWORD-} ]] || [[ -z ${CODE_SERVER_ADDRESS-} ]]; then PASSWORD=e45432jklfdsab CODE_SERVER_ADDRESS=http://localhost:8080 yarn folio --config=config.ts --reporter=list "$@"
echo "The end-to-end testing suites rely on your local environment"
echo -e "\n"
echo "Please set the following environment variables locally:"
echo " \$PASSWORD"
echo " \$CODE_SERVER_ADDRESS"
echo -e "\n"
exit 1
fi
CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts --runInBand
} }
main "$@" main "$@"

73
test/config.ts Normal file
View File

@ -0,0 +1,73 @@
import {
ChromiumEnv,
FirefoxEnv,
WebKitEnv,
test,
setConfig,
PlaywrightOptions,
Config,
globalSetup,
} from "@playwright/test"
import * as crypto from "crypto"
import path from "path"
import { PASSWORD } from "./utils/constants"
import * as wtfnode from "./utils/wtfnode"
// Playwright doesn't like that ../src/node/util has an enum in it
// so I had to copy hash in separately
const hash = (str: string): string => {
return crypto.createHash("sha256").update(str).digest("hex")
}
const cookieToStore = {
sameSite: "Lax" as const,
name: "key",
value: hash(PASSWORD),
domain: "localhost",
path: "/",
expires: -1,
httpOnly: false,
secure: false,
}
globalSetup(async () => {
console.log("\n🚨 Running globalSetup for playwright end-to-end tests")
console.log("👋 Please hang tight...")
if (process.env.WTF_NODE) {
wtfnode.setup()
}
const storage = {
cookies: [cookieToStore],
}
// Save storage state and store as an env variable
// More info: https://playwright.dev/docs/auth?_highlight=authe#reuse-authentication-state
process.env.STORAGE = JSON.stringify(storage)
console.log("✅ globalSetup is now complete.")
})
const config: Config = {
testDir: path.join(__dirname, "e2e"), // Search for tests in this directory.
timeout: 30000, // Each test is given 30 seconds.
retries: 3, // Retry failing tests 2 times
}
if (process.env.CI) {
// In CI, retry failing tests 2 times
// in the event of flakiness
config.retries = 2
}
setConfig(config)
const options: PlaywrightOptions = {
headless: true, // Run tests in headless browsers.
video: "retain-on-failure",
}
// Run tests in three browsers.
test.runWith(new ChromiumEnv(options), { tag: "chromium" })
test.runWith(new FirefoxEnv(options), { tag: "firefox" })
test.runWith(new WebKitEnv(options), { tag: "webkit" })

View File

@ -1,35 +1,15 @@
/// <reference types="jest-playwright-preset" /> import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS } from "../utils/constants"
// This test is for nothing more than to make sure // This is a "gut-check" test to make sure playwright is working as expected
// tests are running in multiple browsers test("browser should display correct userAgent", async ({ page, browserName }) => {
describe("Browser gutcheck", () => {
beforeEach(async () => {
await jestPlaywright.resetBrowser({
logger: {
isEnabled: (name) => name === "browser",
log: (name, severity, message, args) => console.log(`${name} ${message}`),
},
})
})
test("should display correct browser based on userAgent", async () => {
const displayNames = { const displayNames = {
chromium: "Chrome", chromium: "Chrome",
firefox: "Firefox", firefox: "Firefox",
webkit: "Safari", webkit: "Safari",
} }
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
const userAgent = await page.evaluate("navigator.userAgent") const userAgent = await page.evaluate("navigator.userAgent")
if (browserName === "chromium") {
expect(userAgent).toContain(displayNames[browserName]) expect(userAgent).toContain(displayNames[browserName])
}
if (browserName === "firefox") {
expect(userAgent).toContain(displayNames[browserName])
}
if (browserName === "webkit") {
expect(userAgent).toContain(displayNames[browserName])
}
})
}) })

View File

@ -1,20 +1,24 @@
/// <reference types="jest-playwright-preset" /> import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants" import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
// This test is to make sure the globalSetup works as expected // This test is to make sure the globalSetup works as expected
// meaning globalSetup ran and stored the storageState in STORAGE // meaning globalSetup ran and stored the storageState in STORAGE
describe("globalSetup", () => { test.describe("globalSetup", () => {
beforeEach(async () => {
// Create a new context with the saved storage state // Create a new context with the saved storage state
// so we don't have to logged in // so we don't have to logged in
const storageState = JSON.parse(STORAGE) || {} const options: any = {}
await jestPlaywright.resetContext({
storageState,
})
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
})
it("should keep us logged in using the storageState", async () => { // TODO@jsjoeio
// Fix this once https://github.com/microsoft/playwright-test/issues/240
// is fixed
if (STORAGE) {
const storageState = JSON.parse(STORAGE) || {}
options.contextOptions = {
storageState,
}
}
test("should keep us logged in using the storageState", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// Make sure the editor actually loaded // Make sure the editor actually loaded
expect(await page.isVisible("div.monaco-workbench")) expect(await page.isVisible("div.monaco-workbench"))
}) })

View File

@ -1,13 +1,17 @@
/// <reference types="jest-playwright-preset" /> import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants" import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
describe("login", () => { test.describe("login", () => {
beforeEach(async () => { // Reset the browser so no cookies are persisted
await jestPlaywright.resetBrowser() // by emptying the storageState
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" }) const options = {
}) contextOptions: {
storageState: {},
},
}
it("should be able to login", async () => { test("should be able to login", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// Type in password // Type in password
await page.fill(".password", PASSWORD) await page.fill(".password", PASSWORD)
// Click the submit button and login // Click the submit button and login

View File

@ -1,19 +1,17 @@
/// <reference types="jest-playwright-preset" /> import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS } from "../utils/constants" import { CODE_SERVER_ADDRESS } from "../utils/constants"
describe("login page", () => { test.describe("login page", () => {
beforeEach(async () => { // Reset the browser so no cookies are persisted
await jestPlaywright.resetContext({ // by emptying the storageState
logger: { const options = {
isEnabled: (name, severity) => name === "browser", contextOptions: {
log: (name, severity, message, args) => console.log(`${name} ${message}`), storageState: {},
}, },
}) }
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
})
it("should see the login page", async () => { test("should see the login page", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// It should send us to the login page // It should send us to the login page
expect(await page.title()).toBe("code-server login") expect(await page.title()).toBe("code-server login")
}) })

View File

@ -1,13 +1,16 @@
/// <reference types="jest-playwright-preset" /> import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants" import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
describe("logout", () => { test.describe("logout", () => {
beforeEach(async () => { // Reset the browser so no cookies are persisted
await jestPlaywright.resetBrowser() // by emptying the storageState
const options = {
contextOptions: {
storageState: {},
},
}
test("should be able login and logout", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" }) await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
})
it("should be able login and logout", async () => {
// Type in password // Type in password
await page.fill(".password", PASSWORD) await page.fill(".password", PASSWORD)
// Click the submit button and login // Click the submit button and login

View File

@ -1,18 +1,25 @@
/// <reference types="jest-playwright-preset" /> import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants" import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
describe("Open Help > About", () => { test.describe("Open Help > About", () => {
beforeEach(async () => {
// Create a new context with the saved storage state // Create a new context with the saved storage state
// so we don't have to logged in // so we don't have to logged in
const options: any = {}
// TODO@jsjoeio
// Fix this once https://github.com/microsoft/playwright-test/issues/240
// is fixed
if (STORAGE) {
const storageState = JSON.parse(STORAGE) || {} const storageState = JSON.parse(STORAGE) || {}
await jestPlaywright.resetContext({ options.contextOptions = {
storageState, storageState,
}) }
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" }) }
})
it("should see a 'Help' then 'About' button in the Application Menu that opens a dialog", async () => { test(
"should see a 'Help' then 'About' button in the Application Menu that opens a dialog",
options,
async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// Make sure the editor actually loaded // Make sure the editor actually loaded
expect(await page.isVisible("div.monaco-workbench")) expect(await page.isVisible("div.monaco-workbench"))
@ -34,5 +41,6 @@ describe("Open Help > About", () => {
const codeServerText = "text=code-server" const codeServerText = "text=code-server"
expect(await page.isVisible(codeServerText)) expect(await page.isVisible(codeServerText))
}) },
)
}) })

View File

@ -1,39 +0,0 @@
// jest.config.ts
import type { Config } from "@jest/types"
const config: Config.InitialOptions = {
preset: "jest-playwright-preset",
transform: {
"^.+\\.ts$": "<rootDir>/node_modules/ts-jest",
},
globalSetup: "<rootDir>/utils/globalSetup.ts",
testEnvironmentOptions: {
"jest-playwright": {
// TODO(@jsjoeio) enable on webkit and firefox
// waiting on next playwright release
// - https://github.com/microsoft/playwright/issues/6009#event-4536210890
// - https://github.com/microsoft/playwright/issues/6020
browsers: ["chromium"],
// If there's a page error, we don't exit
// i.e. something logged in the console
exitOnPageError: false,
contextOptions: {
recordVideo: {
dir: "./test/e2e/videos",
},
},
},
},
testPathIgnorePatterns: ["/node_modules/", "/lib/", "/out/", "test/unit"],
testTimeout: 30000,
modulePathIgnorePatterns: [
"<rootDir>/../lib/vscode",
"<rootDir>/../release-packages",
"<rootDir>/../release",
"<rootDir>/../release-standalone",
"<rootDir>/../release-npm-package",
"<rootDir>/../release-gcp",
"<rootDir>/../release-images",
],
}
export default config

View File

@ -2,19 +2,19 @@
"license": "MIT", "license": "MIT",
"#": "We must put jest in a sub-directory otherwise VS Code somehow picks up the types and generates conflicts with mocha.", "#": "We must put jest in a sub-directory otherwise VS Code somehow picks up the types and generates conflicts with mocha.",
"devDependencies": { "devDependencies": {
"@playwright/test": "^0.1101.0-alpha2",
"@types/jest": "^26.0.20", "@types/jest": "^26.0.20",
"@types/jsdom": "^16.2.6", "@types/jsdom": "^16.2.6",
"@types/node-fetch": "^2.5.8", "@types/node-fetch": "^2.5.8",
"@types/supertest": "^2.0.10", "@types/supertest": "^2.0.10",
"jest": "^26.6.3", "jest": "^26.6.3",
"jest-playwright-preset": "^1.5.1",
"jsdom": "^16.4.0", "jsdom": "^16.4.0",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"playwright": "^1.8.0", "playwright": "^1.11.0-next-alpha-apr-13-2021",
"playwright-chromium": "^1.10.0",
"playwright-firefox": "^1.10.0",
"playwright-webkit": "^1.10.0",
"supertest": "^6.1.1", "supertest": "^6.1.1",
"ts-jest": "^26.4.4" "ts-jest": "^26.4.4"
},
"resolutions": {
"@playwright/test/playwright": "^1.11.0-next-alpha-apr-13-2021"
} }
} }

File diff suppressed because it is too large Load Diff