import { logger } from "@coder/logger"; /** * Handle for a notification that allows it to be closed and updated. */ export interface INotificationHandle { /** * Closes the notification. */ close(): void; /** * Update the message. */ updateMessage(message: string): void; /** * Update the buttons. */ updateButtons(buttons: INotificationButton[]): void; } /** * Notification severity. */ enum Severity { Ignore = 0, Info = 1, Warning = 2, Error = 3, } /** * Notification button. */ export interface INotificationButton { label: string; run(): void; } /** * Optional notification service. */ export interface INotificationService { /** * Show a notification. */ prompt(severity: Severity, message: string, buttons: INotificationButton[], onCancel: () => void): INotificationHandle; } interface IRetryItem { count?: number; delay?: number; // In seconds. end?: number; // In ms. fn(): any | Promise; // tslint:disable-line no-any can have different return values timeout?: number | NodeJS.Timer; running?: boolean; showInNotification: boolean; } /** * Retry services. Handles multiple services so when a connection drops the * user doesn't get a separate notification for each service. * * Attempts to restart services silently up to a maximum number of tries, then * starts waiting for a delay that grows exponentially with each attempt with a * cap on the delay. Once the delay is long enough, it will show a notification * to the user explaining what is happening with an option to immediately retry. */ export class Retry { private items: Map; // Times are in seconds. private readonly retryMinDelay = 1; private readonly retryMaxDelay = 10; private readonly maxImmediateRetries = 5; private readonly retryExponent = 1.5; private blocked: string | boolean | undefined; private notificationHandle: INotificationHandle | undefined; private updateDelay = 1; private updateTimeout: number | NodeJS.Timer | undefined; private notificationThreshold = 3; // Time in milliseconds to wait before restarting a service. (See usage below // for reasoning.) private waitDelay = 50; public constructor(private notificationService?: INotificationService) { this.items = new Map(); } /** * Set notification service. */ public setNotificationService(notificationService?: INotificationService): void { this.notificationService = notificationService; } /** * Block retries when we know they will fail (for example when starting Wush * back up). If a name is passed, that service will still be allowed to retry * (unless we have already blocked). * * Blocking without a name will override a block with a name. */ public block(name?: string): void { if (!this.blocked || !name) { this.blocked = name || true; this.items.forEach((item) => { this.stopItem(item); }); } } /** * Unblock retries and run any that are pending. */ public unblock(): void { this.blocked = false; this.items.forEach((item, name) => { if (item.running) { this.runItem(name, item); } }); } /** * Register a function to retry that starts/connects to a service. * * If the function returns a promise, it will automatically be retried, * recover, & unblock after calling `run` once (otherwise they need to be * called manually). */ // tslint:disable-next-line no-any can have different return values public register(name: string, fn: () => any | Promise, showInNotification: boolean = true): void { if (this.items.has(name)) { throw new Error(`"${name}" is already registered`); } this.items.set(name, { fn, showInNotification }); } /** * Unregister a function to retry. */ public unregister(name: string): void { if (!this.items.has(name)) { throw new Error(`"${name}" is not registered`); } this.items.delete(name); } /** * Retry a service. */ public run(name: string): void { if (!this.items.has(name)) { throw new Error(`"${name}" is not registered`); } const item = this.items.get(name)!; if (item.running) { throw new Error(`"${name}" is already retrying`); } item.running = true; // This timeout is for the case when the connection drops; this allows time // for the Wush service to come in and block everything because some other // services might make it here first and try to restart, which will fail. setTimeout(() => { if (this.blocked && this.blocked !== name) { return; } if (!item.count || item.count < this.maxImmediateRetries) { return this.runItem(name, item); } if (!item.delay) { item.delay = this.retryMinDelay; } else { item.delay = Math.ceil(item.delay * this.retryExponent); if (item.delay > this.retryMaxDelay) { item.delay = this.retryMaxDelay; } } logger.info(`Retrying ${name.toLowerCase()} in ${item.delay}s`); const itemDelayMs = item.delay * 1000; item.end = Date.now() + itemDelayMs; item.timeout = setTimeout(() => this.runItem(name, item), itemDelayMs); this.updateNotification(); }, this.waitDelay); } /** * Reset a service after a successfully recovering. */ public recover(name: string): void { if (!this.items.has(name)) { throw new Error(`"${name}" is not registered`); } const item = this.items.get(name)!; if (typeof item.timeout === "undefined" && !item.running && typeof item.count !== "undefined") { logger.info(`Recovered connection to ${name.toLowerCase()}`); item.delay = undefined; item.count = undefined; } } /** * Run an item. */ private runItem(name: string, item: IRetryItem): void { if (!item.count) { item.count = 1; } else { ++item.count; } const retryCountText = item.count <= this.maxImmediateRetries ? `[${item.count}/${this.maxImmediateRetries}]` : `[${item.count}]`; logger.info(`Retrying ${name.toLowerCase()} ${retryCountText}...`); const endItem = (): void => { this.stopItem(item); item.running = false; }; try { const maybePromise = item.fn(); if (maybePromise instanceof Promise) { maybePromise.then(() => { endItem(); this.recover(name); if (this.blocked === name) { this.unblock(); } }).catch(() => { endItem(); this.run(name); }); } else { endItem(); } } catch (error) { // Prevent an exception from causing the item to never run again. endItem(); throw error; } } /** * Update, close, or show the notification. */ private updateNotification(): void { if (!this.notificationService) { return; } // tslint:disable-next-line no-any because NodeJS.Timer is valid. clearTimeout(this.updateTimeout as any); const now = Date.now(); const items = Array.from(this.items.entries()).filter(([_, item]) => { return item.showInNotification && typeof item.end !== "undefined" && item.end > now && item.delay && item.delay >= this.notificationThreshold; }).sort((a, b) => { return a[1] < b[1] ? -1 : 1; }); if (items.length === 0) { if (this.notificationHandle) { this.notificationHandle.close(); this.notificationHandle = undefined; } return; } const join = (arr: string[]): string => { const last = arr.pop()!; // Assume length > 0. return arr.length > 0 ? `${arr.join(", ")} and ${last}` : last; }; const servicesStr = join(items.map(([name, _]) => name.toLowerCase())); const message = `Lost connection to ${servicesStr}. Retrying in ${ join(items.map(([_, item]) => `${Math.ceil((item.end! - now) / 1000)}s`)) }.`; const buttons = [{ label: `Retry ${items.length > 1 ? "Services" : items[0][0]} Now`, run: (): void => { logger.info(`Forcing ${servicesStr} to restart now`); items.forEach(([name, item]) => { this.runItem(name, item); }); this.updateNotification(); }, }]; if (!this.notificationHandle) { this.notificationHandle = this.notificationService.prompt( Severity.Info, message, buttons, () => { this.notificationHandle = undefined; // tslint:disable-next-line no-any because NodeJS.Timer is valid. clearTimeout(this.updateTimeout as any); }, ); } else { this.notificationHandle.updateMessage(message); this.notificationHandle.updateButtons(buttons); } this.updateTimeout = setTimeout(() => this.updateNotification(), this.updateDelay * 1000); } /** * Stop an item's timer. */ private stopItem(item: IRetryItem): void { // tslint:disable-next-line no-any because NodeJS.Timer is valid. clearTimeout(item.timeout as any); item.timeout = undefined; item.end = undefined; } } export const retry = new Retry();