diff --git a/assets/js/utils/__tests__/retry.spec.ts b/assets/js/utils/__tests__/retry.spec.ts new file mode 100644 index 000000000..40c155bd3 --- /dev/null +++ b/assets/js/utils/__tests__/retry.spec.ts @@ -0,0 +1,216 @@ +import { mockDateNow, mockRandom } from '../../../test/mock'; +import { retry, RetryFunc, RetryParams } from '../retry'; + +describe('retry', () => { + async function expectRetry(params: RetryParams, maybeFunc?: RetryFunc) { + const func = maybeFunc ?? (() => Promise.reject(new Error('always failing'))); + const spy = vi.fn(func); + + // Preserve the empty name of the anonymous functions. Spy wrapper overrides it. + const funcParam = func.name === '' ? (...args: Parameters>) => spy(...args) : spy; + + const promise = retry(funcParam, params).catch(err => `throw ${err}`); + + await vi.runAllTimersAsync(); + const result = await promise; + + const retries = spy.mock.calls.map(([attempt, nextDelayMs]) => { + const suffix = nextDelayMs === undefined ? '' : 'ms'; + return `${attempt}: ${nextDelayMs}${suffix}`; + }); + + return expect([...retries, result]); + } + + // Remove randomness and real delays from the tests. + mockRandom(); + mockDateNow(0); + + const consoleErrorSpy = vi.spyOn(console, 'error'); + + afterEach(() => { + consoleErrorSpy.mockClear(); + }); + + describe('stops on a successful attempt', () => { + it('first attempt', async () => { + (await expectRetry({}, async () => 'ok')).toMatchInlineSnapshot(` + [ + "1: 200ms", + "ok", + ] + `); + }); + it('middle attempt', async () => { + const func: RetryFunc<'ok'> = async attempt => { + if (attempt !== 2) { + throw new Error('middle failure'); + } + return 'ok'; + }; + + (await expectRetry({}, func)).toMatchInlineSnapshot(` + [ + "1: 200ms", + "2: 300ms", + "ok", + ] + `); + }); + it('last attempt', async () => { + const func: RetryFunc<'ok'> = async attempt => { + if (attempt !== 3) { + throw new Error('last failure'); + } + return 'ok'; + }; + + (await expectRetry({}, func)).toMatchInlineSnapshot(` + [ + "1: 200ms", + "2: 300ms", + "3: undefined", + "ok", + ] + `); + }); + }); + + it('produces a reasonable retry sequence within maxAttempts', async () => { + (await expectRetry({})).toMatchInlineSnapshot(` + [ + "1: 200ms", + "2: 300ms", + "3: undefined", + "throw Error: always failing", + ] + `); + + (await expectRetry({ maxAttempts: 5 })).toMatchInlineSnapshot(` + [ + "1: 200ms", + "2: 300ms", + "3: 600ms", + "4: 1125ms", + "5: undefined", + "throw Error: always failing", + ] + `); + }); + + it('turns into a fixed delay retry algorithm if min/max bounds are equal', async () => { + (await expectRetry({ maxAttempts: 3, minDelayMs: 200, maxDelayMs: 200 })).toMatchInlineSnapshot(` + [ + "1: 200ms", + "2: 200ms", + "3: undefined", + "throw Error: always failing", + ] + `); + }); + + it('allows for zero delay', async () => { + (await expectRetry({ maxAttempts: 3, minDelayMs: 0, maxDelayMs: 0 })).toMatchInlineSnapshot(` + [ + "1: 0ms", + "2: 0ms", + "3: undefined", + "throw Error: always failing", + ] + `); + }); + + describe('fails on first non-retryable error', () => { + it('all errors are retryable', async () => { + (await expectRetry({ isRetryable: () => false })).toMatchInlineSnapshot(` + [ + "1: 200ms", + "throw Error: always failing", + ] + `); + }); + it('middle error is non-retriable', async () => { + const func: RetryFunc = async attempt => { + if (attempt === 3) { + throw new Error('non-retryable'); + } + throw new Error('retryable'); + }; + + const params: RetryParams = { + isRetryable: error => error.message === 'retryable', + }; + + (await expectRetry(params, func)).toMatchInlineSnapshot(` + [ + "1: 200ms", + "2: 300ms", + "3: undefined", + "throw Error: non-retryable", + ] + `); + }); + }); + + it('rejects invalid inputs', async () => { + (await expectRetry({ maxAttempts: 0 })).toMatchInlineSnapshot(` + [ + "throw Error: Invalid 'maxAttempts' for retry: 0", + ] + `); + (await expectRetry({ minDelayMs: -1 })).toMatchInlineSnapshot(` + [ + "throw Error: Invalid 'minDelayMs' for retry: -1", + ] + `); + (await expectRetry({ maxDelayMs: 100 })).toMatchInlineSnapshot(` + [ + "throw Error: Invalid 'maxDelayMs' for retry: 100, 'minDelayMs' is 200", + ] + `); + }); + + it('should use the provided label in logs', async () => { + (await expectRetry({ label: 'test-routine' })).toMatchInlineSnapshot(` + [ + "1: 200ms", + "2: 300ms", + "3: undefined", + "throw Error: always failing", + ] + `); + + expect(consoleErrorSpy.mock.calls).toMatchInlineSnapshot(` + [ + [ + "All 3 attempts of running test-routine failed", + [Error: always failing], + ], + ] + `); + }); + + it('should use the function name in logs', async () => { + async function testFunc() { + throw new Error('always failing'); + } + + (await expectRetry({}, testFunc)).toMatchInlineSnapshot(` + [ + "1: 200ms", + "2: 300ms", + "3: undefined", + "throw Error: always failing", + ] + `); + + expect(consoleErrorSpy.mock.calls).toMatchInlineSnapshot(` + [ + [ + "All 3 attempts of running testFunc failed", + [Error: always failing], + ], + ] + `); + }); +}); diff --git a/assets/js/utils/__tests__/store.spec.ts b/assets/js/utils/__tests__/store.spec.ts index bb8d01685..044f0c4b4 100644 --- a/assets/js/utils/__tests__/store.spec.ts +++ b/assets/js/utils/__tests__/store.spec.ts @@ -2,7 +2,7 @@ import store, { lastUpdatedSuffix } from '../store'; import { mockStorageImpl } from '../../../test/mock-storage'; import { getRandomIntBetween } from '../../../test/randomness'; import { fireEvent } from '@testing-library/dom'; -import { mockDateNow } from '../../../test/mock-date-now'; +import { mockDateNow } from '../../../test/mock'; describe('Store utilities', () => { const { setItemSpy, getItemSpy, removeItemSpy, forceStorageError, setStorageValue } = mockStorageImpl(); diff --git a/assets/js/utils/retry.ts b/assets/js/utils/retry.ts new file mode 100644 index 000000000..bd99ab72f --- /dev/null +++ b/assets/js/utils/retry.ts @@ -0,0 +1,124 @@ +export interface RetryParams { + /** + * Maximum number of attempts to retry the operation. The first attempt counts + * too, so setting this to 1 is equivalent to no retries. + */ + maxAttempts?: number; + + /** + * Initial delay for the first retry. Subsequent retries will be exponentially + * delayed up to `maxDelayMs`. + */ + minDelayMs?: number; + + /** + * Max value a delay can reach. This is useful to avoid unreasonably long + * delays that can be reached at a larger number of retries where the delay + * grows exponentially very fast. + */ + maxDelayMs?: number; + + /** + * If present determines if the error should be retried or immediately re-thrown. + * All errors that aren't instances of `Error` are considered non-retryable. + */ + isRetryable?(error: Error): boolean; + + /** + * Human-readable message to identify the operation being retried. By default + * the function name is used. + */ + label?: string; +} + +export type RetryFunc = (attempt: number, nextDelayMs?: number) => Promise; + +/** + * Retry an async operation with exponential backoff and jitter. + * + * The callback receives the current attempt number and the delay before the + * next attempt in case the current attempt fails. The next delay may be + * `undefined` if this is the last attempt and no further retries will be scheduled. + * + * This is based on the following AWS paper: + * https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + */ +export async function retry(func: RetryFunc, params?: RetryParams): Promise { + const maxAttempts = params?.maxAttempts ?? 3; + + if (maxAttempts < 1) { + throw new Error(`Invalid 'maxAttempts' for retry: ${maxAttempts}`); + } + + const minDelayMs = params?.minDelayMs ?? 200; + + if (minDelayMs < 0) { + throw new Error(`Invalid 'minDelayMs' for retry: ${minDelayMs}`); + } + + const maxDelayMs = params?.maxDelayMs ?? 1500; + + if (maxDelayMs < minDelayMs) { + throw new Error(`Invalid 'maxDelayMs' for retry: ${maxDelayMs}, 'minDelayMs' is ${minDelayMs}`); + } + + const label = params?.label || func.name || '{unnamed routine}'; + + const backoffExponent = 2; + + let attempt = 1; + let nextDelayMs = minDelayMs; + + while (true) { + const hasNextAttempts = attempt < maxAttempts; + + try { + // NB: an `await` is important in this block to make sure the exception is caught + // in this scope. Doing a `return func()` would be a big mistake, so don't try + // to "refactor" that! + return await func(attempt, hasNextAttempts ? nextDelayMs : undefined); + } catch (error) { + if (!(error instanceof Error) || (params?.isRetryable && !params.isRetryable(error))) { + throw error; + } + + if (!hasNextAttempts) { + console.error(`All ${maxAttempts} attempts of running ${label} failed`, error); + throw error; + } + + console.warn( + `[Attempt ${attempt}/${maxAttempts}] Error when running ${label}. Retrying in ${nextDelayMs} milliseconds...`, + error, + ); + + await sleep(nextDelayMs); + + // Equal jitter algorithm taken from AWS blog post's code reference: + // https://github.com/aws-samples/aws-arch-backoff-simulator/blob/66cb169277051eea207dbef8c7f71767fe6af144/src/backoff_simulator.py#L35-L38 + let pure = minDelayMs * backoffExponent ** attempt; + + // Make sure we don't overflow + pure = Math.min(maxDelayMs, pure); + + // Now that we have a purely exponential delay, we add random jitter + // to avoid DDOSing the backend from multiple clients retrying at + // the same time (see the "thundering herd problem" on Wikipedia). + const halfPure = pure / 2; + nextDelayMs = halfPure + randomBetween(0, halfPure); + + // Make sure we don't underflow + nextDelayMs = Math.max(minDelayMs, nextDelayMs); + + attempt += 1; + } + } +} + +function randomBetween(min: number, max: number): number { + return Math.floor(Math.random() * (max - min + 1)) + min; +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} diff --git a/assets/test/mock-date-now.ts b/assets/test/mock-date-now.ts deleted file mode 100644 index fb1627666..000000000 --- a/assets/test/mock-date-now.ts +++ /dev/null @@ -1,9 +0,0 @@ -export function mockDateNow(initialDateNow: number): void { - beforeAll(() => { - vi.useFakeTimers().setSystemTime(initialDateNow); - }); - - afterAll(() => { - vi.useRealTimers(); - }); -} diff --git a/assets/test/mock.ts b/assets/test/mock.ts new file mode 100644 index 000000000..a386efe0c --- /dev/null +++ b/assets/test/mock.ts @@ -0,0 +1,18 @@ +export function mockDateNow(initialDateNow: number): void { + beforeAll(() => { + vi.useFakeTimers().setSystemTime(initialDateNow); + }); + + afterAll(() => { + vi.useRealTimers(); + }); +} + +/** + * Mocks `Math.random` to return a static value. + */ +export function mockRandom(staticValue = 0.5) { + const realRandom = Math.random; + beforeEach(() => (Math.random = () => staticValue)); + afterEach(() => (Math.random = realRandom)); +}