Merge pull request #3169 from cdr/jsjoeio/add-terminal-e2e-test

feat(testing): add e2e tests for code-server and terminal
This commit is contained in:
Joe Previte 2021-04-26 15:16:06 -07:00 committed by GitHub
commit 07d682392e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 314 additions and 49 deletions

View File

@ -1,5 +1,6 @@
import { logger } from "@coder/logger"
import { JSONSchemaForNPMPackageJsonFiles } from "@schemastore/package"
import * as os from "os"
import * as path from "path"
export function getPackageJson(relativePath: string): JSONSchemaForNPMPackageJsonFiles {
@ -18,3 +19,4 @@ const pkg = getPackageJson("../../package.json")
export const version = pkg.version || "development"
export const commit = pkg.commit || "development"
export const rootPath = path.resolve(__dirname, "../..")
export const tmpdir = path.join(os.tmpdir(), "code-server")

View File

@ -4,7 +4,8 @@ import * as path from "path"
import * as tls from "tls"
import { Emitter } from "../common/emitter"
import { generateUuid } from "../common/util"
import { canConnect, tmpdir } from "./util"
import { tmpdir } from "./constants"
import { canConnect } from "./util"
/**
* Provides a way to proxy a TLS socket. Can be used when you need to pass a

View File

@ -8,8 +8,6 @@ import * as path from "path"
import * as util from "util"
import xdgBasedir from "xdg-basedir"
export const tmpdir = path.join(os.tmpdir(), "code-server")
interface Paths {
data: string
config: string

View File

@ -1,15 +1,23 @@
import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS } from "../utils/constants"
import { CodeServer } from "./models/CodeServer"
// This is a "gut-check" test to make sure playwright is working as expected
test("browser should display correct userAgent", async ({ page, browserName }) => {
const displayNames = {
chromium: "Chrome",
firefox: "Firefox",
webkit: "Safari",
}
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
const userAgent = await page.evaluate("navigator.userAgent")
test.describe("browser", () => {
let codeServer: CodeServer
expect(userAgent).toContain(displayNames[browserName])
test.beforeEach(async ({ page }) => {
codeServer = new CodeServer(page)
await codeServer.navigate()
})
test("browser should display correct userAgent", async ({ page, browserName }) => {
const displayNames = {
chromium: "Chrome",
firefox: "Firefox",
webkit: "Safari",
}
const userAgent = await page.evaluate("navigator.userAgent")
expect(userAgent).toContain(displayNames[browserName])
})
})

View File

@ -0,0 +1,45 @@
import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
import { CodeServer } from "./models/CodeServer"
test.describe("CodeServer", () => {
// Create a new context with the saved storage state
// so we don't have to logged in
const options: any = {}
let codeServer: CodeServer
// TODO@jsjoeio
// Fix this once https://github.com/microsoft/playwright-test/issues/240
// is fixed
if (STORAGE) {
const storageState = JSON.parse(STORAGE) || {}
options.contextOptions = {
storageState,
}
}
test.beforeEach(async ({ page }) => {
codeServer = new CodeServer(page)
await codeServer.setup()
})
test(`should navigate to ${CODE_SERVER_ADDRESS}`, options, async ({ page }) => {
// We navigate codeServer before each test
// and we start the test with a storage state
// which means we should be logged in
// so it should be on the address
const url = page.url()
// We use match because there may be a / at the end
// so we don't want it to fail if we expect http://localhost:8080 to match http://localhost:8080/
expect(url).toMatch(CODE_SERVER_ADDRESS)
})
test("should always see the code-server editor", options, async ({ page }) => {
expect(await codeServer.isEditorVisible()).toBe(true)
})
test("should show the Integrated Terminal", options, async ({ page }) => {
await codeServer.focusTerminal()
expect(await page.isVisible("#terminal")).toBe(true)
})
})

View File

@ -1,5 +1,6 @@
import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
import { STORAGE } from "../utils/constants"
import { CodeServer } from "./models/CodeServer"
// This test is to make sure the globalSetup works as expected
// meaning globalSetup ran and stored the storageState in STORAGE
@ -7,6 +8,7 @@ test.describe("globalSetup", () => {
// Create a new context with the saved storage state
// so we don't have to logged in
const options: any = {}
let codeServer: CodeServer
// TODO@jsjoeio
// Fix this once https://github.com/microsoft/playwright-test/issues/240
@ -17,9 +19,12 @@ test.describe("globalSetup", () => {
storageState,
}
}
test.beforeEach(async ({ page }) => {
codeServer = new CodeServer(page)
await codeServer.setup()
})
test("should keep us logged in using the storageState", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// Make sure the editor actually loaded
expect(await page.isVisible("div.monaco-workbench"))
expect(await codeServer.isEditorVisible()).toBe(true)
})
})

View File

@ -1,5 +1,6 @@
import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
import { PASSWORD } from "../utils/constants"
import { CodeServer } from "./models/CodeServer"
test.describe("login", () => {
// Reset the browser so no cookies are persisted
@ -9,26 +10,32 @@ test.describe("login", () => {
storageState: {},
},
}
let codeServer: CodeServer
test.beforeEach(async ({ page }) => {
codeServer = new CodeServer(page)
await codeServer.navigate()
})
test("should see the login page", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// It should send us to the login page
expect(await page.title()).toBe("code-server login")
})
test("should be able to login", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// Type in password
await page.fill(".password", PASSWORD)
// Click the submit button and login
await page.click(".submit")
await page.waitForLoadState("networkidle")
// We do this because occassionally code-server doesn't load on Firefox
// but loads if you reload once or twice
await codeServer.reloadUntilEditorIsVisible()
// Make sure the editor actually loaded
expect(await page.isVisible("div.monaco-workbench"))
expect(await codeServer.isEditorVisible()).toBe(true)
})
test("should see an error message for missing password", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// Skip entering password
// Click the submit button and login
await page.click(".submit")
@ -37,7 +44,6 @@ test.describe("login", () => {
})
test("should see an error message for incorrect password", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// Type in password
await page.fill(".password", "password123")
// Click the submit button and login
@ -47,7 +53,6 @@ test.describe("login", () => {
})
test("should hit the rate limiter for too many unsuccessful logins", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// Type in password
await page.fill(".password", "password123")
// Click the submit button and login

View File

@ -1,5 +1,6 @@
import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
import { CodeServer } from "./models/CodeServer"
test.describe("logout", () => {
// Reset the browser so no cookies are persisted
@ -9,22 +10,31 @@ test.describe("logout", () => {
storageState: {},
},
}
let codeServer: CodeServer
test.beforeEach(async ({ page }) => {
codeServer = new CodeServer(page)
await codeServer.navigate()
})
test("should be able login and logout", options, async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// Type in password
await page.fill(".password", PASSWORD)
// Click the submit button and login
await page.click(".submit")
await page.waitForLoadState("networkidle")
// We do this because occassionally code-server doesn't load on Firefox
// but loads if you reload once or twice
await codeServer.reloadUntilEditorIsVisible()
// Make sure the editor actually loaded
expect(await page.isVisible("div.monaco-workbench"))
expect(await codeServer.isEditorVisible()).toBe(true)
// Click the Application menu
await page.click("[aria-label='Application Menu']")
// See the Log out button
const logoutButton = "a.action-menu-item span[aria-label='Log out']"
expect(await page.isVisible(logoutButton))
expect(await page.isVisible(logoutButton)).toBe(true)
await page.hover(logoutButton)
// TODO(@jsjoeio)

View File

@ -0,0 +1,104 @@
import { Page } from "playwright"
import { CODE_SERVER_ADDRESS } from "../../utils/constants"
// This is a Page Object Model
// We use these to simplify e2e test authoring
// See Playwright docs: https://playwright.dev/docs/pom/
export class CodeServer {
page: Page
constructor(page: Page) {
this.page = page
}
/**
* Navigates to CODE_SERVER_ADDRESS
*/
async navigate() {
await this.page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
}
/**
* Checks if the editor is visible
* and reloads until it is
*/
async reloadUntilEditorIsVisible() {
const editorIsVisible = await this.isEditorVisible()
let reloadCount = 0
// Occassionally code-server timeouts in Firefox
// we're not sure why
// but usually a reload or two fixes it
// TODO@jsjoeio @oxy look into Firefox reconnection/timeout issues
while (!editorIsVisible) {
reloadCount += 1
if (await this.isEditorVisible()) {
console.log(` Editor became visible after ${reloadCount} reloads`)
break
}
// When a reload happens, we want to wait for all resources to be
// loaded completely. Hence why we use that instead of DOMContentLoaded
// Read more: https://thisthat.dev/dom-content-loaded-vs-load/
await this.page.reload({ waitUntil: "load" })
}
}
/**
* Checks if the editor is visible
*/
async isEditorVisible() {
// Make sure the editor actually loaded
// If it's not visible after 5 seconds, something is wrong
await this.page.waitForLoadState("networkidle")
return await this.page.isVisible("div.monaco-workbench", { timeout: 5000 })
}
/**
* Focuses Integrated Terminal
* by going to the Application Menu
* and clicking View > Terminal
*/
async focusTerminal() {
// If the terminal is already visible
// then we can focus it by hitting the keyboard shortcut
const isTerminalVisible = await this.page.isVisible("#terminal")
if (isTerminalVisible) {
await this.page.keyboard.press(`Control+Backquote`)
// Wait for terminal to receive focus
await this.page.waitForSelector("div.terminal.xterm.focus")
// Sometimes the terminal reloads
// which is why we wait for it twice
await this.page.waitForSelector("div.terminal.xterm.focus")
return
}
// Open using the manu
// Click [aria-label="Application Menu"] div[role="none"]
await this.page.click('[aria-label="Application Menu"] div[role="none"]')
// Click text=View
await this.page.hover("text=View")
await this.page.click("text=View")
// Click text=Terminal
await this.page.hover("text=Terminal")
await this.page.click("text=Terminal")
// Wait for terminal to receive focus
// Sometimes the terminal reloads once or twice
// which is why we wait for it to have the focus class
await this.page.waitForSelector("div.terminal.xterm.focus")
// Sometimes the terminal reloads
// which is why we wait for it twice
await this.page.waitForSelector("div.terminal.xterm.focus")
}
/**
* Navigates to CODE_SERVER_ADDRESS
* and reloads until the editor is visible
*
* Helpful for running before tests
*/
async setup() {
await this.navigate()
await this.reloadUntilEditorIsVisible()
}
}

View File

@ -1,10 +1,12 @@
import { test, expect } from "@playwright/test"
import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
import { STORAGE } from "../utils/constants"
import { CodeServer } from "./models/CodeServer"
test.describe("Open Help > About", () => {
// Create a new context with the saved storage state
// so we don't have to logged in
const options: any = {}
let codeServer: CodeServer
// TODO@jsjoeio
// Fix this once https://github.com/microsoft/playwright-test/issues/240
// is fixed
@ -15,32 +17,30 @@ test.describe("Open Help > About", () => {
}
}
test.beforeEach(async ({ page }) => {
codeServer = new CodeServer(page)
await codeServer.setup()
})
test(
"should see a 'Help' then 'About' button in the Application Menu that opens a dialog",
options,
async ({ page }) => {
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
// Make sure the editor actually loaded
expect(await page.isVisible("div.monaco-workbench"))
// Open using the manu
// Click [aria-label="Application Menu"] div[role="none"]
await page.click('[aria-label="Application Menu"] div[role="none"]')
// Click the Application menu
await page.click("[aria-label='Application Menu']")
// See the Help button
const helpButton = "a.action-menu-item span[aria-label='Help']"
expect(await page.isVisible(helpButton))
// Click the Help button
await page.hover("text=Help")
await page.click("text=Help")
// Hover the helpButton
await page.hover(helpButton)
// Click the About button
await page.hover("text=About")
await page.click("text=About")
// see the About button and click it
const aboutButton = "a.action-menu-item span[aria-label='About']"
expect(await page.isVisible(aboutButton))
// NOTE: it won't work unless you hover it first
await page.hover(aboutButton)
await page.click(aboutButton)
const codeServerText = "text=code-server"
expect(await page.isVisible(codeServerText))
// Click div[role="dialog"] >> text=code-server
const element = await page.waitForSelector('div[role="dialog"] >> text=code-server')
expect(element).not.toBeNull()
},
)
})

59
test/e2e/terminal.test.ts Normal file
View File

@ -0,0 +1,59 @@
import { test, expect } from "@playwright/test"
import * as cp from "child_process"
import * as fs from "fs"
// import { tmpdir } from "os"
import * as path from "path"
import util from "util"
import { STORAGE, tmpdir } from "../utils/constants"
import { CodeServer } from "./models/CodeServer"
test.describe("Integrated Terminal", () => {
// Create a new context with the saved storage state
// so we don't have to logged in
const options: any = {}
const testFileName = "pipe"
const testString = "new string test from e2e test"
let codeServer: CodeServer
let tmpFolderPath = ""
let tmpFile = ""
// TODO@jsjoeio
// Fix this once https://github.com/microsoft/playwright-test/issues/240
// is fixed
if (STORAGE) {
const storageState = JSON.parse(STORAGE) || {}
options.contextOptions = {
storageState,
}
}
test.beforeAll(async () => {
tmpFolderPath = await tmpdir("integrated-terminal")
tmpFile = path.join(tmpFolderPath, testFileName)
})
test.beforeEach(async ({ page }) => {
codeServer = new CodeServer(page)
await codeServer.setup()
})
test.afterAll(async () => {
// Ensure directory was removed
await fs.promises.rmdir(tmpFolderPath, { recursive: true })
})
test("should echo a string to a file", options, async ({ page }) => {
const command = `mkfifo '${tmpFile}' && cat '${tmpFile}'`
const exec = util.promisify(cp.exec)
const output = exec(command, { encoding: "utf8" })
// Open terminal and type in value
await codeServer.focusTerminal()
await page.waitForLoadState("load")
await page.keyboard.type(`echo '${testString}' > '${tmpFile}'`)
await page.keyboard.press("Enter")
const { stdout } = await output
expect(stdout).toMatch(testString)
})
})

View File

@ -4,7 +4,8 @@ import * as net from "net"
import * as os from "os"
import * as path from "path"
import { Args, parse, setDefaults, shouldOpenInExistingInstance } from "../../src/node/cli"
import { paths, tmpdir } from "../../src/node/util"
import { tmpdir } from "../../src/node/constants"
import { paths } from "../../src/node/util"
type Mutable<T> = {
-readonly [P in keyof T]: T[P]

View File

@ -1,4 +1,6 @@
import * as fs from "fs"
import { commit, getPackageJson, version } from "../../src/node/constants"
import { tmpdir } from "../../test/utils/constants"
import { loggerModule } from "../utils/helpers"
// jest.mock is hoisted above the imports so we must use `require` here.
@ -51,3 +53,16 @@ describe("constants", () => {
})
})
})
describe("test constants", () => {
describe("tmpdir", () => {
it("should return a temp directory", async () => {
const testName = "temp-dir"
const pathToTempDir = await tmpdir(testName)
expect(pathToTempDir).toContain(testName)
await fs.promises.rmdir(pathToTempDir)
})
})
})

View File

@ -4,8 +4,9 @@ import * as net from "net"
import * as path from "path"
import * as tls from "tls"
import { Emitter } from "../../src/common/emitter"
import { tmpdir } from "../../src/node/constants"
import { SocketProxyProvider } from "../../src/node/socket"
import { generateCertificate, tmpdir } from "../../src/node/util"
import { generateCertificate } from "../../src/node/util"
describe("SocketProxyProvider", () => {
const provider = new SocketProxyProvider()

View File

@ -1,9 +1,9 @@
import { promises as fs } from "fs"
import * as http from "http"
import * as path from "path"
import { tmpdir } from "../../src/node/constants"
import { SettingsProvider, UpdateSettings } from "../../src/node/settings"
import { LatestResponse, UpdateProvider } from "../../src/node/update"
import { tmpdir } from "../../src/node/util"
describe.skip("update", () => {
let version = "1.0.0"

View File

@ -1,3 +1,14 @@
import * as fs from "fs"
import * as os from "os"
import * as path from "path"
export const CODE_SERVER_ADDRESS = process.env.CODE_SERVER_ADDRESS || "http://localhost:8080"
export const PASSWORD = process.env.PASSWORD || "e45432jklfdsab"
export const STORAGE = process.env.STORAGE || ""
export async function tmpdir(testName: string): Promise<string> {
const dir = path.join(os.tmpdir(), "code-server")
await fs.promises.mkdir(dir, { recursive: true })
return await fs.promises.mkdtemp(path.join(dir, `test-${testName}-`), { encoding: "utf8" })
}