Improve CI caching (#416)
* Adjust linux distro to ubuntu 14.04 * Cache lib directory for speedy builds * Fix path linking for default extensions * Update reset * Reset to head * Improve caching * Still run yarn in CI * Update yarn before install * Increase cache timeout * Install vscode from vstar * Undo data-dir changes to CLI, add back clean, remove unused CI func * Remove additional flags added * Remove unused dependency * Reset vscode install dir so patching always works
This commit is contained in:
parent
2683b7c734
commit
dcf409aecb
|
@ -4,3 +4,4 @@ dist
|
|||
out
|
||||
.DS_Store
|
||||
release
|
||||
.cache
|
|
@ -6,11 +6,12 @@ env:
|
|||
matrix:
|
||||
include:
|
||||
- os: linux
|
||||
dist: ubuntu
|
||||
dist: trusty
|
||||
- os: osx
|
||||
before_install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev
|
||||
libsecret-1-dev; fi
|
||||
- npm install -g yarn@1.12.3
|
||||
script:
|
||||
- scripts/build.sh
|
||||
before_deploy:
|
||||
|
@ -35,4 +36,8 @@ deploy:
|
|||
on:
|
||||
repo: codercom/code-server
|
||||
branch: master
|
||||
cache: yarn
|
||||
cache:
|
||||
yarn: true
|
||||
timeout: 1000
|
||||
directories:
|
||||
- .cache
|
||||
|
|
114
build/tasks.ts
114
build/tasks.ts
|
@ -4,21 +4,22 @@ import * as fse from "fs-extra";
|
|||
import * as os from "os";
|
||||
import * as path from "path";
|
||||
import * as zlib from "zlib";
|
||||
import * as https from "https";
|
||||
import * as tar from "tar";
|
||||
|
||||
const isWin = os.platform() === "win32";
|
||||
const libPath = path.join(__dirname, "../lib");
|
||||
const vscodePath = path.join(libPath, "vscode");
|
||||
const defaultExtensionsPath = path.join(libPath, "extensions");
|
||||
const pkgsPath = path.join(__dirname, "../packages");
|
||||
const defaultExtensionsPath = path.join(libPath, "VSCode-linux-x64/resources/app/extensions");
|
||||
const vscodeVersion = process.env.VSCODE_VERSION || "1.32.0";
|
||||
const vsSourceUrl = `https://codesrv-ci.cdr.sh/vstar-${vscodeVersion}.tar.gz`;
|
||||
|
||||
const buildServerBinary = register("build:server:binary", async (runner) => {
|
||||
await ensureInstalled();
|
||||
await copyForDefaultExtensions();
|
||||
await Promise.all([
|
||||
buildBootstrapFork(),
|
||||
buildWeb(),
|
||||
buildDefaultExtensions(),
|
||||
buildServerBundle(),
|
||||
buildAppBrowser(),
|
||||
]);
|
||||
|
@ -129,97 +130,50 @@ const buildWeb = register("build:web", async (runner) => {
|
|||
await runner.execute(isWin ? "npm.cmd" : "npm", ["run", "build"]);
|
||||
});
|
||||
|
||||
const extDirPath = path.join("lib", "vscode-default-extensions");
|
||||
const copyForDefaultExtensions = register("build:copy-vscode", async (runner) => {
|
||||
if (!fs.existsSync(defaultExtensionsPath)) {
|
||||
await ensureClean();
|
||||
await ensureInstalled();
|
||||
await new Promise((resolve, reject): void => {
|
||||
fse.remove(extDirPath, (err) => {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
}
|
||||
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
await new Promise((resolve, reject): void => {
|
||||
fse.copy(vscodePath, extDirPath, (err) => {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
}
|
||||
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const buildDefaultExtensions = register("build:default-extensions", async (runner) => {
|
||||
if (!fs.existsSync(defaultExtensionsPath)) {
|
||||
await copyForDefaultExtensions();
|
||||
runner.cwd = extDirPath;
|
||||
const resp = await runner.execute(isWin ? "npx.cmd" : "npx", [isWin ? "gulp.cmd" : "gulp", "vscode-linux-x64", "--max-old-space-size=32384"]);
|
||||
if (resp.exitCode !== 0) {
|
||||
throw new Error(`Failed to build default extensions: ${resp.stderr}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const ensureInstalled = register("vscode:install", async (runner) => {
|
||||
await ensureCloned();
|
||||
runner.cwd = libPath;
|
||||
|
||||
runner.cwd = vscodePath;
|
||||
const install = await runner.execute(isWin ? "yarn.cmd" : "yarn", []);
|
||||
if (install.exitCode !== 0) {
|
||||
throw new Error(`Failed to install vscode dependencies: ${install.stderr}`);
|
||||
}
|
||||
});
|
||||
if (fs.existsSync(vscodePath) && fs.existsSync(defaultExtensionsPath)) {
|
||||
const pkgVersion = JSON.parse(fs.readFileSync(path.join(vscodePath, "package.json")).toString("utf8")).version;
|
||||
if (pkgVersion === vscodeVersion) {
|
||||
runner.cwd = vscodePath;
|
||||
|
||||
const ensureCloned = register("vscode:clone", async (runner) => {
|
||||
if (fs.existsSync(vscodePath)) {
|
||||
await ensureClean();
|
||||
} else {
|
||||
fse.mkdirpSync(libPath);
|
||||
runner.cwd = libPath;
|
||||
const clone = await runner.execute("git", ["clone", "https://github.com/microsoft/vscode", "--branch", vscodeVersion, "--single-branch", "--depth=1"]);
|
||||
if (clone.exitCode !== 0) {
|
||||
throw new Error(`Failed to clone: ${clone.exitCode}`);
|
||||
const reset = await runner.execute("git", ["reset", "--hard"]);
|
||||
if (reset.exitCode !== 0) {
|
||||
throw new Error(`Failed to clean git repository: ${reset.stderr}`);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
runner.cwd = vscodePath;
|
||||
const checkout = await runner.execute("git", ["checkout", vscodeVersion]);
|
||||
if (checkout.exitCode !== 0) {
|
||||
throw new Error(`Failed to checkout: ${checkout.stderr}`);
|
||||
}
|
||||
});
|
||||
fse.removeSync(libPath);
|
||||
fse.mkdirpSync(libPath);
|
||||
|
||||
const ensureClean = register("vscode:clean", async (runner) => {
|
||||
runner.cwd = vscodePath;
|
||||
await new Promise<void>((resolve, reject): void => {
|
||||
https.get(vsSourceUrl, (res) => {
|
||||
if (res.statusCode !== 200) {
|
||||
return reject(res.statusMessage);
|
||||
}
|
||||
|
||||
const status = await runner.execute("git", ["status", "--porcelain"]);
|
||||
if (status.stdout.trim() !== "") {
|
||||
const clean = await runner.execute("git", ["clean", "-f", "-d", "-X"]);
|
||||
if (clean.exitCode !== 0) {
|
||||
throw new Error(`Failed to clean git repository: ${clean.stderr}`);
|
||||
}
|
||||
const removeUnstaged = await runner.execute("git", ["checkout", "--", "."]);
|
||||
if (removeUnstaged.exitCode !== 0) {
|
||||
throw new Error(`Failed to remove unstaged files: ${removeUnstaged.stderr}`);
|
||||
}
|
||||
}
|
||||
const fetch = await runner.execute("git", ["fetch", "--prune"]);
|
||||
if (fetch.exitCode !== 0) {
|
||||
throw new Error(`Failed to fetch latest changes: ${fetch.stderr}`);
|
||||
}
|
||||
res.pipe(tar.x({
|
||||
C: libPath,
|
||||
}).on("finish", () => {
|
||||
resolve();
|
||||
}).on("error", (err: Error) => {
|
||||
reject(err);
|
||||
}));
|
||||
}).on("error", (err) => {
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const ensurePatched = register("vscode:patch", async (runner) => {
|
||||
if (!fs.existsSync(vscodePath)) {
|
||||
throw new Error("vscode must be cloned to patch");
|
||||
}
|
||||
await ensureClean();
|
||||
await ensureInstalled();
|
||||
|
||||
runner.cwd = vscodePath;
|
||||
const patchPath = path.join(__dirname, "../scripts/vscode.patch");
|
||||
|
|
|
@ -15,7 +15,9 @@
|
|||
"devDependencies": {
|
||||
"@types/fs-extra": "^5.0.4",
|
||||
"@types/node": "^10.12.18",
|
||||
"@types/tar": "^4.0.0",
|
||||
"@types/trash": "^4.3.1",
|
||||
"cache-loader": "^2.0.1",
|
||||
"cross-env": "^5.2.0",
|
||||
"crypto-browserify": "^3.12.0",
|
||||
"css-loader": "^2.1.0",
|
||||
|
@ -34,6 +36,8 @@
|
|||
"sass-loader": "^7.1.0",
|
||||
"string-replace-loader": "^2.1.1",
|
||||
"style-loader": "^0.23.1",
|
||||
"tar": "^4.4.8",
|
||||
"terser-webpack-plugin": "^1.2.3",
|
||||
"ts-loader": "^5.3.3",
|
||||
"ts-node": "^7.0.1",
|
||||
"tsconfig-paths": "^3.8.0",
|
||||
|
|
|
@ -0,0 +1,24 @@
|
|||
#!/bin/bash
|
||||
set -euxo pipefail
|
||||
|
||||
# Builds a tarfile containing vscode sourcefiles neccessary for CI.
|
||||
# Done outside the CI and uploaded to object storage to reduce CI time.
|
||||
|
||||
branch=1.32.0
|
||||
dir=/tmp/vstar
|
||||
outfile=/tmp/vstar-$branch.tar.gz
|
||||
rm -rf $dir
|
||||
mkdir -p $dir
|
||||
|
||||
cd $dir
|
||||
git clone https://github.com/microsoft/vscode --branch $branch --single-branch --depth=1
|
||||
cd vscode
|
||||
|
||||
yarn
|
||||
|
||||
npx gulp vscode-linux-x64 --max-old-space-size=32384
|
||||
rm -rf extensions build out* test
|
||||
cd ..
|
||||
mv *-x64/resources/app/extensions ./extensions
|
||||
rm -rf *-x64
|
||||
tar -czvf $outfile .
|
|
@ -3,6 +3,7 @@ const os = require("os");
|
|||
const environment = process.env.NODE_ENV || "development";
|
||||
const HappyPack = require("happypack");
|
||||
const webpack = require("webpack");
|
||||
const TerserPlugin = require("terser-webpack-plugin");
|
||||
|
||||
const root = path.join(__dirname, "..");
|
||||
|
||||
|
@ -107,6 +108,11 @@ module.exports = (options = {}) => ({
|
|||
id: "ts",
|
||||
threads: Math.max(os.cpus().length - 1, 1),
|
||||
loaders: [{
|
||||
path: "cache-loader",
|
||||
query: {
|
||||
cacheDirectory: path.join(__dirname, "..", ".cache"),
|
||||
},
|
||||
}, {
|
||||
path: "ts-loader",
|
||||
query: {
|
||||
happyPackMode: true,
|
||||
|
@ -121,6 +127,14 @@ module.exports = (options = {}) => ({
|
|||
"process.env.VERSION": `"${process.env.VERSION || ""}"`,
|
||||
}),
|
||||
],
|
||||
optimization: {
|
||||
minimizer: [
|
||||
new TerserPlugin({
|
||||
cache: path.join(__dirname, "..", ".cache", "terser"),
|
||||
parallel: true,
|
||||
}),
|
||||
],
|
||||
},
|
||||
stats: {
|
||||
all: false, // Fallback for options not defined.
|
||||
errors: true,
|
||||
|
|
63
yarn.lock
63
yarn.lock
|
@ -24,6 +24,13 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.12.18.tgz#1d3ca764718915584fcd9f6344621b7672665c67"
|
||||
integrity sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==
|
||||
|
||||
"@types/tar@^4.0.0":
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/@types/tar/-/tar-4.0.0.tgz#e3239d969eeb693a012200613860d0eb871c94f0"
|
||||
integrity sha512-YybbEHNngcHlIWVCYsoj7Oo1JU9JqONuAlt1LlTH/lmL8BMhbzdFUgReY87a05rY1j8mfK47Del+TCkaLAXwLw==
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/trash@^4.3.1":
|
||||
version "4.3.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/trash/-/trash-4.3.1.tgz#4880ff17c4eb467f1a26774ea6328428403b5c57"
|
||||
|
@ -723,6 +730,17 @@ cache-base@^1.0.1:
|
|||
union-value "^1.0.0"
|
||||
unset-value "^1.0.0"
|
||||
|
||||
cache-loader@^2.0.1:
|
||||
version "2.0.1"
|
||||
resolved "https://registry.yarnpkg.com/cache-loader/-/cache-loader-2.0.1.tgz#5758f41a62d7c23941e3c3c7016e6faeb03acb07"
|
||||
integrity sha512-V99T3FOynmGx26Zom+JrVBytLBsmUCzVG2/4NnUKgvXN4bEV42R1ERl1IyiH/cvFIDA1Ytq2lPZ9tXDSahcQpQ==
|
||||
dependencies:
|
||||
loader-utils "^1.1.0"
|
||||
mkdirp "^0.5.1"
|
||||
neo-async "^2.6.0"
|
||||
normalize-path "^3.0.0"
|
||||
schema-utils "^1.0.0"
|
||||
|
||||
camel-case@3.0.x:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-3.0.0.tgz#ca3c3688a4e9cf3a4cda777dc4dcbc713249cf73"
|
||||
|
@ -916,6 +934,11 @@ commander@^2.12.1, commander@^2.18.0:
|
|||
resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a"
|
||||
integrity sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==
|
||||
|
||||
commander@^2.19.0:
|
||||
version "2.20.0"
|
||||
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422"
|
||||
integrity sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==
|
||||
|
||||
commondir@^1.0.1:
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
|
||||
|
@ -3305,7 +3328,7 @@ negotiator@0.6.1:
|
|||
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9"
|
||||
integrity sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=
|
||||
|
||||
neo-async@^2.5.0:
|
||||
neo-async@^2.5.0, neo-async@^2.6.0:
|
||||
version "2.6.0"
|
||||
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.0.tgz#b9d15e4d71c6762908654b5183ed38b753340835"
|
||||
integrity sha512-MFh0d/Wa7vkKO3Y3LlacqAEeHK0mckVqzDieUKTT+KGxi+zIpeVsFxymkIiRpbpDziHc290Xr9A1O4Om7otoRA==
|
||||
|
@ -3452,6 +3475,11 @@ normalize-path@^2.1.1:
|
|||
dependencies:
|
||||
remove-trailing-separator "^1.0.1"
|
||||
|
||||
normalize-path@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
|
||||
integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
|
||||
|
||||
npm-bundled@^1.0.1:
|
||||
version "1.0.5"
|
||||
resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.5.tgz#3c1732b7ba936b3a10325aef616467c0ccbcc979"
|
||||
|
@ -4625,6 +4653,14 @@ source-map-support@^0.5.6, source-map-support@~0.5.6:
|
|||
buffer-from "^1.0.0"
|
||||
source-map "^0.6.0"
|
||||
|
||||
source-map-support@~0.5.10:
|
||||
version "0.5.11"
|
||||
resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.11.tgz#efac2ce0800355d026326a0ca23e162aeac9a4e2"
|
||||
integrity sha512-//sajEx/fGL3iw6fltKMdPvy8kL3kJ2O3iuYlRoT3k9Kb4BjOoZ+BZzaNHeuaruSt+Kf3Zk9tnfAQg9/AJqUVQ==
|
||||
dependencies:
|
||||
buffer-from "^1.0.0"
|
||||
source-map "^0.6.0"
|
||||
|
||||
source-map-url@^0.4.0:
|
||||
version "0.4.0"
|
||||
resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3"
|
||||
|
@ -4924,7 +4960,7 @@ tar@^2.0.0:
|
|||
fstream "^1.0.2"
|
||||
inherits "2"
|
||||
|
||||
tar@^4:
|
||||
tar@^4, tar@^4.4.8:
|
||||
version "4.4.8"
|
||||
resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.8.tgz#b19eec3fde2a96e64666df9fdb40c5ca1bc3747d"
|
||||
integrity sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==
|
||||
|
@ -4951,6 +4987,29 @@ terser-webpack-plugin@^1.1.0:
|
|||
webpack-sources "^1.1.0"
|
||||
worker-farm "^1.5.2"
|
||||
|
||||
terser-webpack-plugin@^1.2.3:
|
||||
version "1.2.3"
|
||||
resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.2.3.tgz#3f98bc902fac3e5d0de730869f50668561262ec8"
|
||||
integrity sha512-GOK7q85oAb/5kE12fMuLdn2btOS9OBZn4VsecpHDywoUC/jLhSAKOiYo0ezx7ss2EXPMzyEWFoE0s1WLE+4+oA==
|
||||
dependencies:
|
||||
cacache "^11.0.2"
|
||||
find-cache-dir "^2.0.0"
|
||||
schema-utils "^1.0.0"
|
||||
serialize-javascript "^1.4.0"
|
||||
source-map "^0.6.1"
|
||||
terser "^3.16.1"
|
||||
webpack-sources "^1.1.0"
|
||||
worker-farm "^1.5.2"
|
||||
|
||||
terser@^3.16.1:
|
||||
version "3.17.0"
|
||||
resolved "https://registry.yarnpkg.com/terser/-/terser-3.17.0.tgz#f88ffbeda0deb5637f9d24b0da66f4e15ab10cb2"
|
||||
integrity sha512-/FQzzPJmCpjAH9Xvk2paiWrFq+5M6aVOf+2KRbwhByISDX/EujxsK+BAvrhb6H+2rtrLCHK9N01wO014vrIwVQ==
|
||||
dependencies:
|
||||
commander "^2.19.0"
|
||||
source-map "~0.6.1"
|
||||
source-map-support "~0.5.10"
|
||||
|
||||
terser@^3.8.1:
|
||||
version "3.14.1"
|
||||
resolved "https://registry.yarnpkg.com/terser/-/terser-3.14.1.tgz#cc4764014af570bc79c79742358bd46926018a32"
|
||||
|
|
Loading…
Reference in New Issue