From 18f395b853cb315f965ea0b2594a595b641f199f Mon Sep 17 00:00:00 2001 From: Forest Hoffman Date: Thu, 21 Mar 2019 14:04:09 -0500 Subject: [PATCH] Fix install from VSIX for TAR and ZIP formats (#245) * Fix install from VSIX for TAR and ZIP formats * Parse TAR before ZIP, when installing from VSIX --- packages/package.json | 11 +- packages/vscode/.gitignore | 3 +- packages/vscode/src/fill/zip.ts | 248 +++++++++++--------- packages/vscode/test/test-extension.tar | Bin 0 -> 10240 bytes packages/vscode/test/test-extension.vsix | Bin 0 -> 1689 bytes packages/vscode/test/zip.test.ts | 59 +++++ packages/vscode/webpack.bootstrap.config.js | 1 + packages/web/webpack.config.js | 1 + 8 files changed, 212 insertions(+), 111 deletions(-) create mode 100644 packages/vscode/test/test-extension.tar create mode 100644 packages/vscode/test/test-extension.vsix create mode 100644 packages/vscode/test/zip.test.ts diff --git a/packages/package.json b/packages/package.json index 72ea26dc..12e48d0b 100644 --- a/packages/package.json +++ b/packages/package.json @@ -12,6 +12,11 @@ "xmlhttprequest": "1.8.0" }, "jest": { + "globals": { + "ts-jest": { + "diagnostics": false + } + }, "moduleFileExtensions": [ "ts", "tsx", @@ -26,7 +31,9 @@ "@coder/ide/src/fill/evaluation": "/ide/src/fill/evaluation", "@coder/ide/src/fill/client": "/ide/src/fill/client", "@coder/(.*)/test": "/$1/test", - "@coder/(.*)": "/$1/src" + "@coder/(.*)": "/$1/src", + "vs/(.*)": "/../lib/vscode/src/vs/$1", + "vszip": "/../lib/vscode/src/vs/base/node/zip.ts" }, "transform": { "^.+\\.tsx?$": "ts-jest" @@ -37,4 +44,4 @@ ], "testRegex": ".*\\.test\\.tsx?" } -} +} \ No newline at end of file diff --git a/packages/vscode/.gitignore b/packages/vscode/.gitignore index c5e82d74..8af4e3b6 100644 --- a/packages/vscode/.gitignore +++ b/packages/vscode/.gitignore @@ -1 +1,2 @@ -bin \ No newline at end of file +bin +test/.test* \ No newline at end of file diff --git a/packages/vscode/src/fill/zip.ts b/packages/vscode/src/fill/zip.ts index 1f6e1cd6..b7aafb20 100644 --- a/packages/vscode/src/fill/zip.ts +++ b/packages/vscode/src/fill/zip.ts @@ -4,11 +4,11 @@ *--------------------------------------------------------------------------------------------*/ import * as nls from "vs/nls"; +import * as vszip from "vszip"; import * as fs from "fs"; import * as path from "path"; import * as tarStream from "tar-stream"; import { promisify } from "util"; -import { ILogService } from "vs/platform/log/common/log"; import { CancellationToken } from "vs/base/common/cancellation"; import { mkdirp } from "vs/base/node/pfs"; @@ -16,8 +16,8 @@ export interface IExtractOptions { overwrite?: boolean; /** - * Source path within the ZIP archive. Only the files contained in this - * path will be extracted. + * Source path within the TAR/ZIP archive. Only the files + * contained in this path will be extracted. */ sourcePath?: string; } @@ -28,11 +28,15 @@ export interface IFile { localPath?: string; } -export function zip(tarPath: string, files: IFile[]): Promise { - return new Promise((c, e) => { +/** + * Override the standard VS Code behavior for zipping + * extensions to use the TAR format instead of ZIP. + */ +export const zip = (tarPath: string, files: IFile[]): Promise => { + return new Promise((c, e): void => { const pack = tarStream.pack(); const chunks: Buffer[] = []; - const ended = new Promise((res, rej) => { + const ended = new Promise((res): void => { pack.on("end", () => { res(Buffer.concat(chunks)); }); @@ -56,132 +60,160 @@ export function zip(tarPath: string, files: IFile[]): Promise { e(ex); }); }); -} +}; -export async function extract(tarPath: string, targetPath: string, options: IExtractOptions = {}, token: CancellationToken): Promise { - const sourcePathRegex = new RegExp(options.sourcePath ? `^${options.sourcePath}` : ''); +/** + * Override the standard VS Code behavior for extracting + * archives, to first attempt to process the archive as a TAR + * and then fallback on the original implementation, for processing + * ZIPs. + */ +export const extract = (archivePath: string, extractPath: string, options: IExtractOptions = {}, token: CancellationToken): Promise => { + return new Promise((c, e): void => { + extractTar(archivePath, extractPath, options, token).then(c).catch((ex) => { + if (!ex.toString().includes("Invalid tar header")) { + e(ex); - return new Promise(async (c, e) => { - const buffer = await promisify(fs.readFile)(tarPath); - const extractor = tarStream.extract(); - extractor.once('error', e); - extractor.on('entry', (header, stream, next) => { - const rawName = header.name; - - const nextEntry = (): void => { - stream.resume(); - next(); - }; - - if (token.isCancellationRequested) { - return nextEntry(); - } - - if (!sourcePathRegex.test(rawName)) { - return nextEntry(); - } - - const fileName = rawName.replace(sourcePathRegex, ''); - - const targetFileName = path.join(targetPath, fileName); - if (/\/$/.test(fileName)) { - stream.resume(); - mkdirp(targetFileName).then(() => { - next(); - }, e); return; } - - const dirName = path.dirname(fileName); - const targetDirName = path.join(targetPath, dirName); - if (targetDirName.indexOf(targetPath) !== 0) { - e(nls.localize('invalid file', "Error extracting {0}. Invalid file.", fileName)); - return nextEntry(); - } - - mkdirp(targetDirName, void 0, token).then(() => { - const fstream = fs.createWriteStream(targetFileName, { mode: header.mode }); - fstream.once('close', () => { - next(); - }); - fstream.once('error', (err) => { - e(err); - }); - stream.pipe(fstream); - stream.resume(); - }); + vszip.extract(archivePath, extractPath, options, token).then(c).catch(e); }); - extractor.once('finish', () => { - c(); - }); - extractor.write(buffer); - extractor.end(); }); -} +}; -export function buffer(tarPath: string, filePath: string): Promise { - return new Promise(async (c, e) => { +/** + * Override the standard VS Code behavior for buffering + * archives, to first process the Buffer as a TAR and then + * fallback on the original implementation, for processing ZIPs. + */ +export const buffer = (targetPath: string, filePath: string): Promise => { + return new Promise((c, e): void => { let done: boolean = false; - extractAssets(tarPath, new RegExp(filePath), (path: string, data: Buffer) => { - if (path === filePath) { + extractAssets(targetPath, new RegExp(filePath), (assetPath: string, data: Buffer) => { + if (path.normalize(assetPath) === path.normalize(filePath)) { done = true; c(data); } }).then(() => { if (!done) { - e("couldnt find asset " + filePath); + e("couldn't find asset " + filePath); } }).catch((ex) => { - e(ex); + if (!ex.toString().includes("Invalid tar header")) { + e(ex); + + return; + } + vszip.buffer(targetPath, filePath).then(c).catch(e); }); }); -} +}; -async function extractAssets(tarPath: string, match: RegExp, callback: (path: string, data: Buffer) => void): Promise { - const buffer = await promisify(fs.readFile)(tarPath); - const extractor = tarStream.extract(); - let callbackResolve: () => void; - let callbackReject: (ex?) => void; - const complete = new Promise((r, rej) => { - callbackResolve = r; - callbackReject = rej; - }); - extractor.once("error", (err) => { - callbackReject(err); - }); - extractor.on("entry", (header, stream, next) => { - const name = header.name; - if (match.test(name)) { - extractData(stream).then((data) => { - callback(name, data); - next(); +/** + * Override the standard VS Code behavior for extracting assets + * from archive Buffers to use the TAR format instead of ZIP. + */ +export const extractAssets = (tarPath: string, match: RegExp, callback: (path: string, data: Buffer) => void): Promise => { + return new Promise(async (c, e): Promise => { + try { + const buffer = await promisify(fs.readFile)(tarPath); + const extractor = tarStream.extract(); + extractor.once("error", e); + extractor.on("entry", (header, stream, next) => { + const name = header.name; + if (match.test(name)) { + extractData(stream).then((data) => { + callback(name, data); + next(); + }).catch(e); + stream.resume(); + } else { + stream.on("end", () => { + next(); + }); + stream.resume(); + } }); - stream.resume(); - } else { - stream.on("end", () => { - next(); + extractor.on("finish", () => { + c(); }); - stream.resume(); + extractor.write(buffer); + extractor.end(); + } catch (ex) { + e(ex); } }); - extractor.on("finish", () => { - callbackResolve(); - }); - extractor.write(buffer); - extractor.end(); - return complete; -} +}; -async function extractData(stream: NodeJS.ReadableStream): Promise { - return new Promise((res, rej) => { +const extractData = (stream: NodeJS.ReadableStream): Promise => { + return new Promise((c, e): void => { const fileData: Buffer[] = []; - stream.on('data', (data) => fileData.push(data)); - stream.on('end', () => { + stream.on("data", (data) => fileData.push(data)); + stream.on("end", () => { const fd = Buffer.concat(fileData); - res(fd); - }); - stream.on('error', (err) => { - rej(err); + c(fd); }); + stream.on("error", e); }); -} +}; + +const extractTar = (tarPath: string, targetPath: string, options: IExtractOptions = {}, token: CancellationToken): Promise => { + return new Promise(async (c, e): Promise => { + try { + const sourcePathRegex = new RegExp(options.sourcePath ? `^${options.sourcePath}` : ""); + const buffer = await promisify(fs.readFile)(tarPath); + const extractor = tarStream.extract(); + extractor.once("error", e); + extractor.on("entry", (header, stream, next) => { + const rawName = path.normalize(header.name); + + const nextEntry = (): void => { + stream.resume(); + next(); + }; + + if (token.isCancellationRequested) { + return nextEntry(); + } + + if (!sourcePathRegex.test(rawName)) { + return nextEntry(); + } + + const fileName = rawName.replace(sourcePathRegex, ""); + const targetFileName = path.join(targetPath, fileName); + if (/\/$/.test(fileName)) { + stream.resume(); + mkdirp(targetFileName).then(() => { + next(); + }, e); + + return; + } + + const dirName = path.dirname(fileName); + const targetDirName = path.join(targetPath, dirName); + if (targetDirName.indexOf(targetPath) !== 0) { + e(nls.localize("invalid file", "Error extracting {0}. Invalid file.", fileName)); + + return nextEntry(); + } + + return mkdirp(targetDirName, undefined, token).then(() => { + const fstream = fs.createWriteStream(targetFileName, { mode: header.mode }); + fstream.once("close", () => { + next(); + }); + fstream.once("error", e); + stream.pipe(fstream); + stream.resume(); + }); + }); + extractor.once("finish", c); + extractor.write(buffer); + extractor.end(); + } catch (ex) { + e(ex); + } + }); +}; diff --git a/packages/vscode/test/test-extension.tar b/packages/vscode/test/test-extension.tar new file mode 100644 index 0000000000000000000000000000000000000000..bd1f69c39ee458ca843fba191705e8e8f3b781e0 GIT binary patch literal 10240 zcmeHLZExE)5cb#kS8yEAwZJU-MUppHcF`tXW+2%DBe=mk<-3u=uhkoPUf=h%~O z-T>l17zTlj|38sh&G^Sm>E}p-hs59a1Bkyj9C$YV|4RHfbm081p2z_c|G=yGez=SO zL00p>?*)B`|8US9v{3hn?BA;9f6o7N5f>SC=rR{lD>Oz)!6Oab;N>gysr^+XxDqg) ziS#9&G5qRv`|5T3RzU6GmxMr*|8o*ul9WQ>i(Mz6y|;B&{SW%v`VWR(sQ;kv1OMCl z-y8CKFZ|R-2y-&0c!cnV`|)>1yr9yY3(ZK+={ns?8dDWXmK%;RBb9JYSou~SAvS6t zld9org@XCU9;nuE)-EkZGl4$h_fiTuLR=s-fK*N+mM|KB#OTEfM3+pX9)^`yrC+3g zbGqCb&sYQ}*m4Kcvjg0|y7%tw#76x)3&oaj`C$pf;3<*0qyD|F-&^mzl|~FonOx|nu0ypDeWjK0Rg<-R7H2_w7AHa zno)Uuo?B}^WGc_d>StijNkdQQHU}yv4!5Lf_Bv9~Yr&BmzBw4%crw-BDnvMNU40w> z8q{nT)fUf5swS6ob-}m^kP6*WQrS2lnx;ZBswTf^a5{7wl4{Bk$ucTeKW0Wx=w5rR zX1pVD3JOBE{!>X$r3srhs!IXDT8maCiYJ>me_*O0*_ke4CY%ER*LlxV#wmbHvI4gJ zThHnLV6v=6@j$EoFLET$D9t@=%&$Yb0a?nElKiQEk4GX&P@3+Q;-}>w9{^ z^iRdbQ+V)@9KIQWA~!dshkD^Mr3#a)56k)+#*ZnH{BNckkv1u?w~cp_l&k_}o5AsF zU9jKKBkaE_zSw{pnd=V literal 0 HcmV?d00001 diff --git a/packages/vscode/test/test-extension.vsix b/packages/vscode/test/test-extension.vsix new file mode 100644 index 0000000000000000000000000000000000000000..3c133799fed977f6c04d857ceb00f5145efec40e GIT binary patch literal 1689 zcmWIWW@h1H0D*ITd46C9l;C2JVMwhgNzE(H%+J#g4dG;9UVOVNX34#-n9>Sv21b^z zj0_AcB0$vva5FZdnZb`@MoMOJ3B;g0LIzC)8g!bw@vkY+co3GxGzjiCy{uxe>x_Vg zCSbZQH$SB`Csi-Cq9DJhq*%dLA+0noxdiAZ4NZk=F3#lqyyASId`^D4hEhhRlBP9R ztu+_OOCYatFx&vTL`O23r60(<3dCY4E-6S%&Q45E1v()=5A28?*d5{8zmc!mfXAhL z-=R+jLY8jW6eep`p4!=@9W#~X=rSj#Kh@4bTOaTJeD`EDYr{Ri7FmV0UU3!`HnKl_ z4ZELgS!U?Fkh!zl=Jj>Wf=zR-9$54|zPHkAqqozgqtjBDycShn>}9Olb4((M-R?kQ z!P4w`)7DCOO<$*Eg&=X0~&FC-q-|2eAn_9=hfckZy?> z$${oQy|UuWirmD!%(T?v60rB4-N72hYeT(@FB=H#`yMV){6J*2ruYkscV8G)jX6uy z7P_a2WG8oPOmf$5Z;L;FYQ2Rfv&q90@2J}Q@0Qmd3HQ$y%{j2hq)LQu@n-{uxl)Ry zW?bIeW%h8dpS1Z$<8tkJ2d8B3WGOkAT>V(d>U%kd>tl^GbG7v(51X``9?#GX&c2`% zJ7t!s@CL?It_?ZO{>``AV(0L;pZ)4x$a27e;grvHi<6N8M}iAt3XcD6nwHFX%X2%g zT1>j?Q3EbV{(a{TCHT~cXz~c?XbQY*lDg(-JB3@-kbQUlaj7TPovn{=j$U*!;gJV?HfBwsljajIJYUmyLuHs?0a zo%>cUwiX?a8Fb!l-+b$t)WocHiL1m8)jDBR89!VYKwsIbFo0j^RD*#dE( zJ#Z8CQOhu(*`P8EkJ*?-2C~`Tkj+LbK!9e03J?slS=oR-W?%)veL&quSwK7hLfuM( literal 0 HcmV?d00001 diff --git a/packages/vscode/test/zip.test.ts b/packages/vscode/test/zip.test.ts new file mode 100644 index 00000000..e7685dc4 --- /dev/null +++ b/packages/vscode/test/zip.test.ts @@ -0,0 +1,59 @@ +import * as zip from "../src/fill/zip"; +import * as path from "path"; +import * as fs from "fs"; +import * as cp from "child_process"; +import { CancellationToken } from "vs/base/common/cancellation"; + +// tslint:disable-next-line:no-any +jest.mock("vs/nls", () => ({ "localize": (...args: any): string => `${JSON.stringify(args)}` })); + +describe("zip", () => { + const tarPath = path.resolve(__dirname, "./test-extension.tar"); + const vsixPath = path.resolve(__dirname, "./test-extension.vsix"); + const extractPath = path.resolve(__dirname, "./.test-extension"); + + beforeEach(() => { + if (!fs.existsSync(extractPath) || path.dirname(extractPath) !== __dirname) { + return; + } + cp.execSync(`rm -rf '${extractPath}'`); + }); + + const resolveExtract = async (archivePath: string): Promise => { + expect(fs.existsSync(archivePath)).toEqual(true); + await expect(zip.extract( + archivePath, + extractPath, + { sourcePath: "extension", overwrite: true }, + CancellationToken.None, + )).resolves.toBe(undefined); + expect(fs.existsSync(extractPath)).toEqual(true); + }; + + // tslint:disable-next-line:no-any + const extract = (archivePath: string): () => any => { + // tslint:disable-next-line:no-any + return async (): Promise => { + await resolveExtract(archivePath); + expect(fs.existsSync(path.resolve(extractPath, ".vsixmanifest"))).toEqual(true); + expect(fs.existsSync(path.resolve(extractPath, "package.json"))).toEqual(true); + }; + }; + it("should extract from tarred VSIX", extract(tarPath), 2000); + it("should extract from zipped VSIX", extract(vsixPath), 2000); + + // tslint:disable-next-line:no-any + const buffer = (archivePath: string): () => any => { + // tslint:disable-next-line:no-any + return async (): Promise => { + await resolveExtract(archivePath); + const manifestPath = path.resolve(extractPath, ".vsixmanifest"); + expect(fs.existsSync(manifestPath)).toEqual(true); + const manifestBuf = fs.readFileSync(manifestPath); + expect(manifestBuf.length).toBeGreaterThan(0); + await expect(zip.buffer(archivePath, "extension.vsixmanifest")).resolves.toEqual(manifestBuf); + }; + }; + it("should buffer tarred VSIX", buffer(tarPath), 2000); + it("should buffer zipped VSIX", buffer(vsixPath), 2000); +}); diff --git a/packages/vscode/webpack.bootstrap.config.js b/packages/vscode/webpack.bootstrap.config.js index abfda9bd..0b50f54e 100644 --- a/packages/vscode/webpack.bootstrap.config.js +++ b/packages/vscode/webpack.bootstrap.config.js @@ -62,6 +62,7 @@ module.exports = merge( "vs/platform/product/node/package": path.resolve(vsFills, "package.ts"), "vs/platform/product/node/product": path.resolve(vsFills, "product.ts"), "vs/base/node/zip": path.resolve(vsFills, "zip.ts"), + "vszip": path.resolve(root, "lib/vscode/src/vs/base/node/zip.ts"), "vs": path.resolve(root, "lib/vscode/src/vs"), }, }, diff --git a/packages/web/webpack.config.js b/packages/web/webpack.config.js index ecf027dd..8c0316ac 100644 --- a/packages/web/webpack.config.js +++ b/packages/web/webpack.config.js @@ -75,6 +75,7 @@ module.exports = merge( "vs/platform/product/node/package": path.resolve(vsFills, "package.ts"), "vs/platform/product/node/product": path.resolve(vsFills, "product.ts"), "vs/base/node/zip": path.resolve(vsFills, "zip.ts"), + "vszip": path.resolve(root, "lib/vscode/src/vs/base/node/zip.ts"), "vs": path.join(root, "lib", "vscode", "src", "vs"), }, },