code-server/ci/build.ts

503 lines
17 KiB
TypeScript
Raw Normal View History

2020-02-04 19:27:46 +00:00
import { Binary } from "@coder/nbin"
import * as cp from "child_process"
import * as fs from "fs-extra"
import * as os from "os"
import Bundler from "parcel-bundler"
import * as path from "path"
import * as util from "util"
enum Task {
2020-02-04 19:27:46 +00:00
Binary = "binary",
Package = "package",
Build = "build",
Watch = "watch",
}
class Builder {
2020-02-04 19:27:46 +00:00
private readonly rootPath = path.resolve(__dirname, "..")
private readonly vscodeSourcePath = path.join(this.rootPath, "lib/vscode")
private readonly binariesPath = path.join(this.rootPath, "binaries")
private readonly buildPath = path.join(this.rootPath, "build")
private readonly codeServerVersion: string
private _target?: "darwin" | "alpine" | "linux"
private currentTask?: Task
public constructor() {
this.ensureArgument("rootPath", this.rootPath)
this.codeServerVersion = this.ensureArgument(
"codeServerVersion",
2020-02-15 00:46:00 +00:00
process.env.VERSION || require(path.join(this.rootPath, "package.json")).version,
2020-02-04 19:27:46 +00:00
)
}
public run(task: Task | undefined): void {
this.currentTask = task
this.doRun(task).catch((error) => {
console.error(error.message)
process.exit(1)
})
}
private async task<T>(message: string, fn: () => Promise<T>): Promise<T> {
const time = Date.now()
this.log(`${message}...`, true)
try {
const t = await fn()
process.stdout.write(`took ${Date.now() - time}ms\n`)
return t
} catch (error) {
process.stdout.write("failed\n")
throw error
}
}
/**
* Writes to stdout with an optional newline.
*/
private log(message: string, skipNewline = false): void {
process.stdout.write(`[${this.currentTask || "default"}] ${message}`)
if (!skipNewline) {
process.stdout.write("\n")
}
}
private async doRun(task: Task | undefined): Promise<void> {
if (!task) {
throw new Error("No task provided")
}
const arch = this.ensureArgument("arch", os.arch().replace(/^x/, "x86_"))
const target = this.ensureArgument("target", await this.target())
const binaryName = `code-server-${this.codeServerVersion}-${target}-${arch}`
switch (task) {
case Task.Watch:
return this.watch()
case Task.Binary:
return this.binary(binaryName)
case Task.Package:
return this.package(binaryName)
case Task.Build:
return this.build()
default:
throw new Error(`No task matching "${task}"`)
}
}
/**
* Get the target of the system.
*/
private async target(): Promise<"darwin" | "alpine" | "linux"> {
if (!this._target) {
if (os.platform() === "darwin" || (process.env.OSTYPE && /^darwin/.test(process.env.OSTYPE))) {
this._target = "darwin"
} else {
// Alpine's ldd doesn't have a version flag but if you use an invalid flag
// (like --version) it outputs the version to stderr and exits with 1.
const result = await util
.promisify(cp.exec)("ldd --version")
.catch((error) => ({ stderr: error.message, stdout: "" }))
if (/musl/.test(result.stderr) || /musl/.test(result.stdout)) {
this._target = "alpine"
} else {
this._target = "linux"
}
}
}
return this._target
}
/**
* Make sure the argument is set. Display the value if it is.
*/
private ensureArgument(name: string, arg?: string): string {
if (!arg) {
throw new Error(`${name} is missing`)
}
this.log(`${name} is "${arg}"`)
return arg
}
/**
* Build VS Code and code-server.
*/
private async build(): Promise<void> {
process.env.NODE_OPTIONS = "--max-old-space-size=32384 " + (process.env.NODE_OPTIONS || "")
process.env.NODE_ENV = "production"
await this.task("cleaning up old build", async () => {
if (!process.env.SKIP_VSCODE) {
return fs.remove(this.buildPath)
}
// If skipping VS Code, keep the existing build if any.
try {
const files = await fs.readdir(this.buildPath)
return Promise.all(files.filter((f) => f !== "lib").map((f) => fs.remove(path.join(this.buildPath, f))))
} catch (error) {
if (error.code !== "ENOENT") {
throw error
}
}
})
const commit = require(path.join(this.vscodeSourcePath, "build/lib/util")).getVersion(this.rootPath) as string
if (!process.env.SKIP_VSCODE) {
await this.buildVscode(commit)
} else {
this.log("skipping vs code build")
}
await this.buildCodeServer(commit)
this.log(`final build: ${this.buildPath}`)
}
private async buildCodeServer(commit: string): Promise<void> {
await this.task("building code-server", async () => {
return util.promisify(cp.exec)("tsc --outDir ./out-build --tsBuildInfoFile ./.prod.tsbuildinfo", {
cwd: this.rootPath,
})
})
await this.task("bundling code-server", async () => {
return this.createBundler("dist-build", commit).bundle()
})
await this.task("copying code-server into build directory", async () => {
await fs.mkdirp(this.buildPath)
await Promise.all([
fs.copy(path.join(this.rootPath, "out-build"), path.join(this.buildPath, "out")),
fs.copy(path.join(this.rootPath, "dist-build"), path.join(this.buildPath, "dist")),
// For source maps and images.
fs.copy(path.join(this.rootPath, "src"), path.join(this.buildPath, "src")),
])
})
await this.copyDependencies("code-server", this.rootPath, this.buildPath, {
commit,
version: this.codeServerVersion,
2020-02-05 00:16:45 +00:00
})
2020-02-04 19:27:46 +00:00
}
private async buildVscode(commit: string): Promise<void> {
await this.task("building vs code", () => {
return util.promisify(cp.exec)("yarn gulp compile-build", { cwd: this.vscodeSourcePath })
})
await this.task("building builtin extensions", async () => {
const exists = await fs.pathExists(path.join(this.vscodeSourcePath, ".build/extensions"))
if (exists) {
process.stdout.write("already built, skipping...")
} else {
await util.promisify(cp.exec)("yarn gulp compile-extensions-build", { cwd: this.vscodeSourcePath })
}
})
await this.task("optimizing vs code", async () => {
return util.promisify(cp.exec)("yarn gulp optimize --gulpfile ./coder.js", { cwd: this.vscodeSourcePath })
})
if (process.env.MINIFY) {
await this.task("minifying vs code", () => {
return util.promisify(cp.exec)("yarn gulp minify --gulpfile ./coder.js", { cwd: this.vscodeSourcePath })
})
}
const vscodeBuildPath = path.join(this.buildPath, "lib/vscode")
await this.task("copying vs code into build directory", async () => {
await fs.mkdirp(path.join(vscodeBuildPath, "resources/linux"))
2020-02-04 19:27:46 +00:00
await Promise.all([
fs.move(
path.join(this.vscodeSourcePath, `out-vscode${process.env.MINIFY ? "-min" : ""}`),
2020-02-15 00:46:00 +00:00
path.join(vscodeBuildPath, "out"),
),
fs.copy(path.join(this.vscodeSourcePath, ".build/extensions"), path.join(vscodeBuildPath, "extensions")),
fs.copy(
path.join(this.vscodeSourcePath, "resources/linux/code.png"),
path.join(vscodeBuildPath, "resources/linux/code.png"),
),
2020-02-04 19:27:46 +00:00
])
})
await this.copyDependencies("vs code", this.vscodeSourcePath, vscodeBuildPath, {
commit,
date: new Date().toISOString(),
2020-02-04 19:27:46 +00:00
})
}
private async copyDependencies(name: string, sourcePath: string, buildPath: string, merge: object): Promise<void> {
2020-02-04 19:27:46 +00:00
await this.task(`copying ${name} dependencies`, async () => {
return Promise.all(
["node_modules", "package.json", "yarn.lock"].map((fileName) => {
return fs.copy(path.join(sourcePath, fileName), path.join(buildPath, fileName))
2020-02-15 00:46:00 +00:00
}),
2020-02-04 19:27:46 +00:00
)
})
const fileName = name === "code-server" ? "package" : "product"
await this.task(`writing final ${name} ${fileName}.json`, async () => {
const json = JSON.parse(await fs.readFile(path.join(sourcePath, `${fileName}.json`), "utf8"))
return fs.writeFile(
path.join(buildPath, `${fileName}.json`),
JSON.stringify(
{
...json,
...merge,
},
null,
2020-02-15 00:46:00 +00:00
2,
),
)
})
2020-02-04 19:27:46 +00:00
if (process.env.MINIFY) {
await this.task(`restricting ${name} to production dependencies`, async () => {
await util.promisify(cp.exec)("yarn --production --ignore-scripts", { cwd: buildPath })
2020-02-04 19:27:46 +00:00
})
}
}
/**
* Bundles the built code into a binary.
*/
private async binary(binaryName: string): Promise<void> {
const prependCode = async (code: string, relativeFilePath: string): Promise<void> => {
const filePath = path.join(this.buildPath, relativeFilePath)
const content = await fs.readFile(filePath, "utf8")
if (!content.startsWith(code)) {
await fs.writeFile(filePath, code + content)
}
}
// Unpack binaries since we can't run them within the binary.
const unpack = `
if (global.NBIN_LOADED) {
try {
const fs = require("fs-extra")
const rg = require("vscode-ripgrep")
const path = require("path")
const { logger, field } = require("@coder/logger")
const unpackExecutables = async (filePath, destination) => {
logger.debug("unpacking executable", field("src", filePath), field("dest", destination))
await fs.mkdirp(path.dirname(destination))
if (filePath && !(await fs.pathExists(destination))) {
await fs.writeFile(destination, await fs.readFile(filePath))
await fs.chmod(destination, "755")
}
}
unpackExecutables(rg.binaryRgPath, rg.rgPath).catch((error) => console.warn(error))
} catch (error) {
console.warn(error)
}
}
`
// Enable finding files within the binary.
const loader = `
if (!global.NBIN_LOADED) {
try {
const nbin = require("nbin")
nbin.shimNativeFs("${this.buildPath}")
global.NBIN_LOADED = true
require("@coder/logger").logger.debug("shimmed file system at ${this.buildPath}")
const path = require("path")
const rg = require("vscode-ripgrep")
rg.binaryRgPath = rg.rgPath
rg.rgPath = path.join(require("os").tmpdir(), "code-server/binaries", path.basename(rg.binaryRgPath))
} catch (error) {
// Most likely not in the binary.
}
}
`
await this.task("Prepending nbin loader", () => {
return Promise.all([
prependCode(loader, "out/node/entry.js"),
prependCode(loader, "lib/vscode/out/vs/server/entry.js"),
prependCode(loader + unpack, "lib/vscode/out/vs/server/fork.js"),
prependCode(loader, "lib/vscode/out/bootstrap-fork.js"),
prependCode(loader, "lib/vscode/extensions/node_modules/typescript/lib/tsserver.js"),
])
})
2020-02-04 19:27:46 +00:00
const bin = new Binary({
mainFile: path.join(this.buildPath, "out/node/entry.js"),
target: await this.target(),
})
bin.writeFiles(path.join(this.buildPath, "**"))
await fs.mkdirp(this.binariesPath)
const binaryPath = path.join(this.binariesPath, binaryName)
await fs.writeFile(binaryPath, await bin.build())
await fs.chmod(binaryPath, "755")
this.log(`binary: ${binaryPath}`)
}
/**
* Package the binary into a release archive.
*/
private async package(binaryName: string): Promise<void> {
const releasePath = path.join(this.rootPath, "release")
const archivePath = path.join(releasePath, binaryName)
await fs.remove(archivePath)
await fs.mkdirp(archivePath)
await fs.copyFile(path.join(this.binariesPath, binaryName), path.join(archivePath, "code-server"))
await fs.copyFile(path.join(this.rootPath, "README.md"), path.join(archivePath, "README.md"))
await fs.copyFile(path.join(this.vscodeSourcePath, "LICENSE.txt"), path.join(archivePath, "LICENSE.txt"))
await fs.copyFile(
path.join(this.vscodeSourcePath, "ThirdPartyNotices.txt"),
2020-02-15 00:46:00 +00:00
path.join(archivePath, "ThirdPartyNotices.txt"),
2020-02-04 19:27:46 +00:00
)
if ((await this.target()) === "darwin") {
await util.promisify(cp.exec)(`zip -r "${binaryName}.zip" "${binaryName}"`, { cwd: releasePath })
this.log(`archive: ${archivePath}.zip`)
} else {
await util.promisify(cp.exec)(`tar -czf "${binaryName}.tar.gz" "${binaryName}"`, { cwd: releasePath })
this.log(`archive: ${archivePath}.tar.gz`)
}
}
private async watch(): Promise<void> {
let server: cp.ChildProcess | undefined
const restartServer = (): void => {
if (server) {
server.kill()
}
2020-02-07 00:26:07 +00:00
const s = cp.fork(path.join(this.rootPath, "out/node/entry.js"), process.argv.slice(3))
2020-02-04 19:27:46 +00:00
console.log(`[server] spawned process ${s.pid}`)
s.on("exit", () => console.log(`[server] process ${s.pid} exited`))
server = s
}
const vscode = cp.spawn("yarn", ["watch"], { cwd: this.vscodeSourcePath })
const tsc = cp.spawn("tsc", ["--watch", "--pretty", "--preserveWatchOutput"], { cwd: this.rootPath })
const bundler = this.createBundler()
const cleanup = (code?: number | null): void => {
this.log("killing vs code watcher")
vscode.removeAllListeners()
vscode.kill()
this.log("killing tsc")
tsc.removeAllListeners()
tsc.kill()
if (server) {
this.log("killing server")
server.removeAllListeners()
server.kill()
}
this.log("killing bundler")
process.exit(code || 0)
}
process.on("SIGINT", () => cleanup())
process.on("SIGTERM", () => cleanup())
vscode.on("exit", (code) => {
this.log("vs code watcher terminated unexpectedly")
cleanup(code)
})
tsc.on("exit", (code) => {
this.log("tsc terminated unexpectedly")
cleanup(code)
})
const bundle = bundler.bundle().catch(() => {
this.log("parcel watcher terminated unexpectedly")
cleanup(1)
})
bundler.on("buildEnd", () => {
console.log("[parcel] bundled")
})
2020-02-05 00:16:45 +00:00
bundler.on("buildError", (error) => {
console.error("[parcel]", error)
})
2020-02-04 19:27:46 +00:00
vscode.stderr.on("data", (d) => process.stderr.write(d))
tsc.stderr.on("data", (d) => process.stderr.write(d))
// From https://github.com/chalk/ansi-regex
const pattern = [
"[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)",
"(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))",
].join("|")
const re = new RegExp(pattern, "g")
/**
* Split stdout on newlines and strip ANSI codes.
*/
const onLine = (proc: cp.ChildProcess, callback: (strippedLine: string, originalLine: string) => void): void => {
let buffer = ""
if (!proc.stdout) {
throw new Error("no stdout")
}
proc.stdout.setEncoding("utf8")
proc.stdout.on("data", (d) => {
const data = buffer + d
const split = data.split("\n")
const last = split.length - 1
for (let i = 0; i < last; ++i) {
callback(split[i].replace(re, ""), split[i])
}
// The last item will either be an empty string (the data ended with a
// newline) or a partial line (did not end with a newline) and we must
// wait to parse it until we get a full line.
buffer = split[last]
})
}
let startingVscode = false
onLine(vscode, (line, original) => {
console.log("[vscode]", original)
// Wait for watch-client since "Finished compilation" will appear multiple
// times before the client starts building.
if (!startingVscode && line.includes("Starting watch-client")) {
startingVscode = true
} else if (startingVscode && line.includes("Finished compilation") && process.env.AUTO_PATCH) {
cp.exec("yarn patch:generate", { cwd: this.rootPath }, (error, _, stderr) => {
if (error || stderr) {
console.error(error ? error.message : stderr)
}
})
}
})
onLine(tsc, (line, original) => {
// tsc outputs blank lines; skip them.
if (line !== "") {
console.log("[tsc]", original)
}
if (line.includes("Watching for file changes")) {
bundle.then(restartServer)
}
})
}
private createBundler(out = "dist", commit?: string): Bundler {
return new Bundler(path.join(this.rootPath, "src/browser/pages/app.ts"), {
2020-02-04 19:27:46 +00:00
cache: true,
cacheDir: path.join(this.rootPath, ".cache"),
detailedReport: true,
minify: !!process.env.MINIFY,
hmr: false,
logLevel: 1,
outDir: path.join(this.rootPath, out),
publicUrl: `/static-${commit}/dist`,
target: "browser",
})
}
}
2020-02-04 19:27:46 +00:00
const builder = new Builder()
builder.run(process.argv[2] as Task)