[Part 4] Add a retry utility for frontend

assets/js/utils/__tests__/retry.spec.ts

@@ -0,0 +1,216 @@
+import { mockDateNow, mockRandom } from '../../../test/mock';
+import { retry, RetryFunc, RetryParams } from '../retry';
+
+describe('retry', () => {
+  async function expectRetry<R>(params: RetryParams, maybeFunc?: RetryFunc<R>) {
+    const func = maybeFunc ?? (() => Promise.reject(new Error('always failing')));
+    const spy = vi.fn(func);
+
+    // Preserve the empty name of the anonymous functions. Spy wrapper overrides it.
+    const funcParam = func.name === '' ? (...args: Parameters<RetryFunc<R>>) => spy(...args) : spy;
+
+    const promise = retry(funcParam, params).catch(err => `throw ${err}`);
+
+    await vi.runAllTimersAsync();
+    const result = await promise;
+
+    const retries = spy.mock.calls.map(([attempt, nextDelayMs]) => {
+      const suffix = nextDelayMs === undefined ? '' : 'ms';
+      return `${attempt}: ${nextDelayMs}${suffix}`;
+    });
+
+    return expect([...retries, result]);
+  }
+
+  // Remove randomness and real delays from the tests.
+  mockRandom();
+  mockDateNow(0);
+
+  const consoleErrorSpy = vi.spyOn(console, 'error');
+
+  afterEach(() => {
+    consoleErrorSpy.mockClear();
+  });
+
+  describe('stops on a successful attempt', () => {
+    it('first attempt', async () => {
+      (await expectRetry({}, async () => 'ok')).toMatchInlineSnapshot(`
+        [
+          "1: 200ms",
+          "ok",
+        ]
+      `);
+    });
+    it('middle attempt', async () => {
+      const func: RetryFunc<'ok'> = async attempt => {
+        if (attempt !== 2) {
+          throw new Error('middle failure');
+        }
+        return 'ok';
+      };
+
+      (await expectRetry({}, func)).toMatchInlineSnapshot(`
+        [
+          "1: 200ms",
+          "2: 300ms",
+          "ok",
+        ]
+      `);
+    });
+    it('last attempt', async () => {
+      const func: RetryFunc<'ok'> = async attempt => {
+        if (attempt !== 3) {
+          throw new Error('last failure');
+        }
+        return 'ok';
+      };
+
+      (await expectRetry({}, func)).toMatchInlineSnapshot(`
+        [
+          "1: 200ms",
+          "2: 300ms",
+          "3: undefined",
+          "ok",
+        ]
+      `);
+    });
+  });
+
+  it('produces a reasonable retry sequence within maxAttempts', async () => {
+    (await expectRetry({})).toMatchInlineSnapshot(`
+      [
+        "1: 200ms",
+        "2: 300ms",
+        "3: undefined",
+        "throw Error: always failing",
+      ]
+    `);
+
+    (await expectRetry({ maxAttempts: 5 })).toMatchInlineSnapshot(`
+      [
+        "1: 200ms",
+        "2: 300ms",
+        "3: 600ms",
+        "4: 1125ms",
+        "5: undefined",
+        "throw Error: always failing",
+      ]
+    `);
+  });
+
+  it('turns into a fixed delay retry algorithm if min/max bounds are equal', async () => {
+    (await expectRetry({ maxAttempts: 3, minDelayMs: 200, maxDelayMs: 200 })).toMatchInlineSnapshot(`
+      [
+        "1: 200ms",
+        "2: 200ms",
+        "3: undefined",
+        "throw Error: always failing",
+      ]
+    `);
+  });
+
+  it('allows for zero delay', async () => {
+    (await expectRetry({ maxAttempts: 3, minDelayMs: 0, maxDelayMs: 0 })).toMatchInlineSnapshot(`
+      [
+        "1: 0ms",
+        "2: 0ms",
+        "3: undefined",
+        "throw Error: always failing",
+      ]
+    `);
+  });
+
+  describe('fails on first non-retryable error', () => {
+    it('all errors are retryable', async () => {
+      (await expectRetry({ isRetryable: () => false })).toMatchInlineSnapshot(`
+        [
+          "1: 200ms",
+          "throw Error: always failing",
+        ]
+      `);
+    });
+    it('middle error is non-retriable', async () => {
+      const func: RetryFunc<never> = async attempt => {
+        if (attempt === 3) {
+          throw new Error('non-retryable');
+        }
+        throw new Error('retryable');
+      };
+
+      const params: RetryParams = {
+        isRetryable: error => error.message === 'retryable',
+      };
+
+      (await expectRetry(params, func)).toMatchInlineSnapshot(`
+        [
+          "1: 200ms",
+          "2: 300ms",
+          "3: undefined",
+          "throw Error: non-retryable",
+        ]
+      `);
+    });
+  });
+
+  it('rejects invalid inputs', async () => {
+    (await expectRetry({ maxAttempts: 0 })).toMatchInlineSnapshot(`
+      [
+        "throw Error: Invalid 'maxAttempts' for retry: 0",
+      ]
+    `);
+    (await expectRetry({ minDelayMs: -1 })).toMatchInlineSnapshot(`
+      [
+        "throw Error: Invalid 'minDelayMs' for retry: -1",
+      ]
+    `);
+    (await expectRetry({ maxDelayMs: 100 })).toMatchInlineSnapshot(`
+      [
+        "throw Error: Invalid 'maxDelayMs' for retry: 100, 'minDelayMs' is 200",
+      ]
+    `);
+  });
+
+  it('should use the provided label in logs', async () => {
+    (await expectRetry({ label: 'test-routine' })).toMatchInlineSnapshot(`
+      [
+        "1: 200ms",
+        "2: 300ms",
+        "3: undefined",
+        "throw Error: always failing",
+      ]
+    `);
+
+    expect(consoleErrorSpy.mock.calls).toMatchInlineSnapshot(`
+      [
+        [
+          "All 3 attempts of running test-routine failed",
+          [Error: always failing],
+        ],
+      ]
+    `);
+  });
+
+  it('should use the function name in logs', async () => {
+    async function testFunc() {
+      throw new Error('always failing');
+    }
+
+    (await expectRetry({}, testFunc)).toMatchInlineSnapshot(`
+      [
+        "1: 200ms",
+        "2: 300ms",
+        "3: undefined",
+        "throw Error: always failing",
+      ]
+    `);
+
+    expect(consoleErrorSpy.mock.calls).toMatchInlineSnapshot(`
+      [
+        [
+          "All 3 attempts of running testFunc failed",
+          [Error: always failing],
+        ],
+      ]
+    `);
+  });
+});

assets/js/utils/__tests__/store.spec.ts

@@ -2,7 +2,7 @@ import store, { lastUpdatedSuffix } from '../store';
 import { mockStorageImpl } from '../../../test/mock-storage';
 import { getRandomIntBetween } from '../../../test/randomness';
 import { fireEvent } from '@testing-library/dom';
-import { mockDateNow } from '../../../test/mock-date-now';
+import { mockDateNow } from '../../../test/mock';
 
 describe('Store utilities', () => {
   const { setItemSpy, getItemSpy, removeItemSpy, forceStorageError, setStorageValue } = mockStorageImpl();

assets/js/utils/retry.ts

@@ -0,0 +1,124 @@
+export interface RetryParams {
+  /**
+   * Maximum number of attempts to retry the operation. The first attempt counts
+   * too, so setting this to 1 is equivalent to no retries.
+   */
+  maxAttempts?: number;
+
+  /**
+   * Initial delay for the first retry. Subsequent retries will be exponentially
+   * delayed up to `maxDelayMs`.
+   */
+  minDelayMs?: number;
+
+  /**
+   * Max value a delay can reach. This is useful to avoid unreasonably long
+   * delays that can be reached at a larger number of retries where the delay
+   * grows exponentially very fast.
+   */
+  maxDelayMs?: number;
+
+  /**
+   * If present determines if the error should be retried or immediately re-thrown.
+   * All errors that aren't instances of `Error` are considered non-retryable.
+   */
+  isRetryable?(error: Error): boolean;
+
+  /**
+   * Human-readable message to identify the operation being retried. By default
+   * the function name is used.
+   */
+  label?: string;
+}
+
+export type RetryFunc<R = void> = (attempt: number, nextDelayMs?: number) => Promise<R>;
+
+/**
+ * Retry an async operation with exponential backoff and jitter.
+ *
+ * The callback receives the current attempt number and the delay before the
+ * next attempt in case the current attempt fails. The next delay may be
+ * `undefined` if this is the last attempt and no further retries will be scheduled.
+ *
+ * This is based on the following AWS paper:
+ * https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
+ */
+export async function retry<R>(func: RetryFunc<R>, params?: RetryParams): Promise<R> {
+  const maxAttempts = params?.maxAttempts ?? 3;
+
+  if (maxAttempts < 1) {
+    throw new Error(`Invalid 'maxAttempts' for retry: ${maxAttempts}`);
+  }
+
+  const minDelayMs = params?.minDelayMs ?? 200;
+
+  if (minDelayMs < 0) {
+    throw new Error(`Invalid 'minDelayMs' for retry: ${minDelayMs}`);
+  }
+
+  const maxDelayMs = params?.maxDelayMs ?? 1500;
+
+  if (maxDelayMs < minDelayMs) {
+    throw new Error(`Invalid 'maxDelayMs' for retry: ${maxDelayMs}, 'minDelayMs' is ${minDelayMs}`);
+  }
+
+  const label = params?.label || func.name || '{unnamed routine}';
+
+  const backoffExponent = 2;
+
+  let attempt = 1;
+  let nextDelayMs = minDelayMs;
+
+  while (true) {
+    const hasNextAttempts = attempt < maxAttempts;
+
+    try {
+      // NB: an `await` is important in this block to make sure the exception is caught
+      // in this scope. Doing a `return func()` would be a big mistake, so don't try
+      // to "refactor" that!
+      return await func(attempt, hasNextAttempts ? nextDelayMs : undefined);
+    } catch (error) {
+      if (!(error instanceof Error) || (params?.isRetryable && !params.isRetryable(error))) {
+        throw error;
+      }
+
+      if (!hasNextAttempts) {
+        console.error(`All ${maxAttempts} attempts of running ${label} failed`, error);
+        throw error;
+      }
+
+      console.warn(
+        `[Attempt ${attempt}/${maxAttempts}] Error when running ${label}. Retrying in ${nextDelayMs} milliseconds...`,
+        error,
+      );
+
+      await sleep(nextDelayMs);
+
+      // Equal jitter algorithm taken from AWS blog post's code reference:
+      // https://github.com/aws-samples/aws-arch-backoff-simulator/blob/66cb169277051eea207dbef8c7f71767fe6af144/src/backoff_simulator.py#L35-L38
+      let pure = minDelayMs * backoffExponent ** attempt;
+
+      // Make sure we don't overflow
+      pure = Math.min(maxDelayMs, pure);
+
+      // Now that we have a purely exponential delay, we add random jitter
+      // to avoid DDOSing the backend from multiple clients retrying at
+      // the same time (see the "thundering herd problem" on Wikipedia).
+      const halfPure = pure / 2;
+      nextDelayMs = halfPure + randomBetween(0, halfPure);
+
+      // Make sure we don't underflow
+      nextDelayMs = Math.max(minDelayMs, nextDelayMs);
+
+      attempt += 1;
+    }
+  }
+}
+
+function randomBetween(min: number, max: number): number {
+  return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

assets/test/mock.ts

@@ -0,0 +1,18 @@
+export function mockDateNow(initialDateNow: number): void {
+  beforeAll(() => {
+    vi.useFakeTimers().setSystemTime(initialDateNow);
+  });
+
+  afterAll(() => {
+    vi.useRealTimers();
+  });
+}
+
+/**
+ * Mocks `Math.random` to return a static value.
+ */
+export function mockRandom(staticValue = 0.5) {
+  const realRandom = Math.random;
+  beforeEach(() => (Math.random = () => staticValue));
+  afterEach(() => (Math.random = realRandom));
+}