import { exec } from "child_process"; import { appendFile } from "fs"; import { promisify } from "util"; import { logger, Logger } from "@coder/logger"; import { escapePath } from "@coder/server"; import { IURI } from "./uri"; /** * Represents an uploadable directory, so we can query for existing files once. */ interface IUploadableDirectory { existingFiles: string[]; filesToUpload: Map; preparePromise?: Promise; } /** * There doesn't seem to be a provided type for entries, so here is an * incomplete version. */ interface IEntry { name: string; isFile: boolean; file: (cb: (file: File) => void) => void; createReader: () => ({ readEntries: (cb: (entries: Array) => void) => void; }); } /** * Updatable progress. */ interface IProgress { /** * Report progress. Progress is the completed percentage from 0 to 100. */ report(progress: number): void; } /** * Service for reporting progress. */ interface IProgressService { /** * Start a new progress bar that resolves & disappears when the task finishes. */ start(title:string, task: (progress: IProgress) => Promise): Promise; } /** * Service for notifications. */ interface INotificationService { /** * Display an error message. */ error(error: Error): void; /** * Ask for a decision. */ prompt(message: string, choices: string[]): Promise; } /** * Handles file uploads. */ export class Upload { private readonly maxParallelUploads = 100; private readonly readSize = 32000; // ~32kb max while reading in the file. private readonly packetSize = 32000; // ~32kb max when writing. private readonly notificationService: INotificationService; private readonly progressService: IProgressService; private readonly logger: Logger; private readonly currentlyUploadingFiles: Map; private readonly queueByDirectory: Map; private progress: IProgress | undefined; private uploadPromise: Promise | undefined; private resolveUploadPromise: (() => void) | undefined; private finished: number; private uploadedFilePaths: string[]; private total: number; public constructor(notificationService: INotificationService, progressService: IProgressService) { this.notificationService = notificationService; this.progressService = progressService; this.logger = logger.named("Upload"); this.currentlyUploadingFiles = new Map(); this.queueByDirectory = new Map(); this.uploadedFilePaths = []; this.finished = 0; this.total = 0; } /** * Upload dropped files. This will try to upload everything it can. Errors * will show via notifications. If an upload operation is ongoing, the files * will be added to that operation. */ public async uploadDropped(event: DragEvent, uploadDir: IURI): Promise { this.addDirectory(uploadDir.path); await this.queueFiles(event, uploadDir); this.logger.debug( // -1 so we don't include the uploadDir itself. `Uploading ${this.queueByDirectory.size - 1} directories and ${this.total} files`, ); await this.prepareDirectories(); if (!this.uploadPromise) { this.uploadPromise = this.progressService.start("Uploading files...", (progress) => { return new Promise((resolve): void => { this.progress = progress; this.resolveUploadPromise = (): void => { const uploaded = this.uploadedFilePaths; this.uploadPromise = undefined; this.resolveUploadPromise = undefined; this.uploadedFilePaths = []; this.finished = 0; this.total = 0; resolve(uploaded); }; }); }); } this.uploadFiles(); return this.uploadPromise; } /** * Cancel all file uploads. */ public async cancel(): Promise { this.currentlyUploadingFiles.clear(); this.queueByDirectory.clear(); } /** * Create directories and get existing files. * On failure, show the error and remove the failed directory from the queue. */ private async prepareDirectories(): Promise { await Promise.all(Array.from(this.queueByDirectory).map(([path, dir]) => { if (!dir.preparePromise) { dir.preparePromise = this.prepareDirectory(path, dir); } return dir.preparePromise; })); } /** * Create a directory and get existing files. * On failure, show the error and remove the directory from the queue. */ private async prepareDirectory(path: string, dir: IUploadableDirectory): Promise { await Promise.all([ promisify(exec)(`mkdir -p ${escapePath(path)}`).catch((error) => { const message = error.message.toLowerCase(); if (message.includes("file exists")) { throw new Error(`Unable to create directory at ${path} because a file exists there`); } throw new Error(error.message || `Unable to upload ${path}`); }), // Only get files, so we don't show an override option that will just // fail anyway. promisify(exec)(`find ${escapePath(path)} -maxdepth 1 -not -type d`).then((stdio) => { dir.existingFiles = stdio.stdout.split("\n"); }), ]).catch((error) => { this.queueByDirectory.delete(path); this.notificationService.error(error); }); } /** * Upload as many files as possible. When finished, resolve the upload promise. */ private uploadFiles(): void { const finishFileUpload = (path: string): void => { ++this.finished; this.currentlyUploadingFiles.delete(path); this.progress!.report(Math.floor((this.finished / this.total) * 100)); this.uploadFiles(); }; while (this.queueByDirectory.size > 0 && this.currentlyUploadingFiles.size < this.maxParallelUploads) { const [dirPath, dir] = this.queueByDirectory.entries().next().value; if (dir.filesToUpload.size === 0) { this.queueByDirectory.delete(dirPath); continue; } const [filePath, item] = dir.filesToUpload.entries().next().value; this.currentlyUploadingFiles.set(filePath, item); dir.filesToUpload.delete(filePath); this.uploadFile(filePath, item, dir.existingFiles).then(() => { finishFileUpload(filePath); }).catch((error) => { this.notificationService.error(error); finishFileUpload(filePath); }); } if (this.queueByDirectory.size === 0 && this.currentlyUploadingFiles.size === 0) { this.resolveUploadPromise!(); } } /** * Upload a file. */ private async uploadFile(path: string, file: File, existingFiles: string[]): Promise { if (existingFiles.includes(path)) { const choice = await this.notificationService.prompt(`${path} already exists. Overwrite?`, ["Yes", "No"]); if (choice !== "Yes") { return; } } await new Promise(async (resolve, reject): Promise => { let readOffset = 0; const reader = new FileReader(); const seek = (): void => { const slice = file.slice(readOffset, readOffset + this.readSize); readOffset += this.readSize; reader.readAsArrayBuffer(slice); }; const rm = async (): Promise => { await promisify(exec)(`rm -f ${escapePath(path)}`); }; await rm(); reader.addEventListener("load", async () => { const buffer = new Uint8Array(reader.result as ArrayBuffer); let bufferOffset = 0; while (bufferOffset <= buffer.length) { // Got canceled while sending data. if (!this.currentlyUploadingFiles.has(path)) { await rm(); return resolve(); } const data = buffer.slice(bufferOffset, bufferOffset + this.packetSize); try { await promisify(appendFile)(path, data); } catch (error) { await rm(); const message = error.message.toLowerCase(); if (message.includes("no space")) { return reject(new Error("You are out of disk space")); } else if (message.includes("is a directory")) { return reject(new Error(`Unable to upload ${path} because there is a directory there`)); } return reject(new Error(error.message || `Unable to upload ${path}`)); } bufferOffset += this.packetSize; } if (readOffset >= file.size) { this.uploadedFilePaths.push(path); return resolve(); } seek(); }); seek(); }); } /** * Queue files from a drop event. We have to get the files first; we can't do * it in tandem with uploading or the entries will disappear. */ private async queueFiles(event: DragEvent, uploadDir: IURI): Promise { if (!event.dataTransfer || !event.dataTransfer.items) { return; } const promises: Array> = []; for (let i = 0; i < event.dataTransfer.items.length; i++) { const item = event.dataTransfer.items[i]; if (typeof item.webkitGetAsEntry === "function") { promises.push(this.traverseItem(item.webkitGetAsEntry(), uploadDir.fsPath).catch(this.notificationService.error)); } else { const file = item.getAsFile(); if (file) { this.addFile(uploadDir.fsPath, uploadDir.fsPath + "/" + file.name, file); } } } await Promise.all(promises); } /** * Traverses an entry and add files to the queue. */ private async traverseItem(entry: IEntry, parentPath: string): Promise { if (entry.isFile) { return new Promise((resolve): void => { entry.file((file) => { this.addFile( parentPath, parentPath + "/" + file.name, file, ); resolve(); }); }); } parentPath += "/" + entry.name; this.addDirectory(parentPath); await new Promise((resolve): void => { const promises: Array> = []; const dirReader = entry.createReader(); // According to the spec, readEntries() must be called until it calls // the callback with an empty array. const readEntries = (): void => { dirReader.readEntries((entries) => { if (entries.length === 0) { Promise.all(promises).then(resolve).catch((error) => { this.notificationService.error(error); resolve(); }); } else { promises.push(...entries.map((child) => this.traverseItem(child, parentPath))); readEntries(); } }); }; readEntries(); }); } /** * Add a file to the queue. */ private addFile(parentPath: string, path: string, file: File): void { ++this.total; this.addDirectory(parentPath); this.queueByDirectory.get(parentPath)!.filesToUpload.set(path, file); } /** * Add a directory to the queue. */ private addDirectory(path: string): void { if (!this.queueByDirectory.has(path)) { this.queueByDirectory.set(path, { existingFiles: [], filesToUpload: new Map(), }); } } }