From 3d251b341a56a763184a0a15657ebc1bf6aaa60a Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 6 Apr 2023 14:36:31 +0100 Subject: [PATCH 01/35] Add host port wait strategy to static helper --- .../docker-compose-environment.ts | 10 +++--- src/generic-container/generic-container.ts | 18 ++++------- .../started-generic-container.ts | 2 +- src/wait-for-container.ts | 3 +- src/wait-strategy/default-wait-strategy.ts | 11 ------- src/wait-strategy/host-port-wait-strategy.ts | 32 +++++++++++++------ src/wait-strategy/http-wait-strategy.ts | 4 ++- src/wait-strategy/log-wait-strategy.ts | 7 +--- src/wait-strategy/wait-strategy.ts | 3 +- src/wait-strategy/wait.ts | 5 +++ 10 files changed, 45 insertions(+), 50 deletions(-) delete mode 100644 src/wait-strategy/default-wait-strategy.ts diff --git a/src/docker-compose-environment/docker-compose-environment.ts b/src/docker-compose-environment/docker-compose-environment.ts index d6654a169..b1578ad62 100644 --- a/src/docker-compose-environment/docker-compose-environment.ts +++ b/src/docker-compose-environment/docker-compose-environment.ts @@ -16,7 +16,7 @@ import { StartedDockerComposeEnvironment } from "./started-docker-compose-enviro import { dockerComposeDown } from "../docker-compose/functions/docker-compose-down"; import { dockerComposeUp } from "../docker-compose/functions/docker-compose-up"; import { waitForContainer } from "../wait-for-container"; -import { defaultWaitStrategy } from "../wait-strategy/default-wait-strategy"; +import { Wait } from "../wait-strategy/wait"; import { DefaultPullPolicy, PullPolicy } from "../pull-policy"; import { dockerComposePull } from "../docker-compose/functions/docker-compose-pull"; @@ -131,13 +131,11 @@ export class DockerComposeEnvironment { const container = await getContainerById(startedContainer.Id); const containerName = resolveContainerName(this.projectName, startedContainer.Names[0]); - const { dockerode, provider, host, hostIps } = await dockerClient(); + const { host, hostIps } = await dockerClient(); const inspectResult = await inspectContainer(container); const boundPorts = BoundPorts.fromInspectResult(hostIps, inspectResult); const waitStrategy = ( - this.waitStrategy[containerName] - ? this.waitStrategy[containerName] - : defaultWaitStrategy(host, dockerode, provider, container) + this.waitStrategy[containerName] ? this.waitStrategy[containerName] : Wait.forListeningPorts() ).withStartupTimeout(this.startupTimeout); if (containerLog.enabled()) { @@ -148,7 +146,7 @@ export class DockerComposeEnvironment { try { log.info(`Waiting for container ${containerName} to be ready`); - await waitForContainer(container, waitStrategy, host, boundPorts); + await waitForContainer(container, waitStrategy, boundPorts); log.info(`Container ${containerName} is ready`); } catch (err) { log.error(`Container ${containerName} failed to be ready: ${err}`); diff --git a/src/generic-container/generic-container.ts b/src/generic-container/generic-container.ts index 3ff64451f..1cc8f9812 100644 --- a/src/generic-container/generic-container.ts +++ b/src/generic-container/generic-container.ts @@ -38,7 +38,7 @@ import { LABEL_TESTCONTAINERS_CONTAINER_HASH } from "../labels"; import { StartedNetwork } from "../network"; import { waitForContainer } from "../wait-for-container"; import { initCreateContainerOptions } from "./create-container-options"; -import { defaultWaitStrategy } from "../wait-strategy/default-wait-strategy"; +import { Wait } from "../wait-strategy/wait"; const reusableContainerCreationLock = new AsyncLock(); @@ -128,18 +128,16 @@ export class GenericContainer implements TestContainer { } private async reuseContainer(container: Dockerode.Container) { - const { dockerode, provider, host, hostIps } = await dockerClient(); + const { host, hostIps } = await dockerClient(); const inspectResult = await inspectContainer(container); const boundPorts = BoundPorts.fromInspectResult(hostIps, inspectResult).filter(this.opts.exposedPorts); - const waitStrategy = ( - this.waitStrategy ?? defaultWaitStrategy(host, dockerode, provider, container) - ).withStartupTimeout(this.startupTimeout); + const waitStrategy = (this.waitStrategy ?? Wait.forListeningPorts()).withStartupTimeout(this.startupTimeout); if (this.containerStarting) { await this.containerStarting(inspectResult, true); } - await waitForContainer(container, waitStrategy, host, boundPorts); + await waitForContainer(container, waitStrategy, boundPorts); const startedContainer = new StartedGenericContainer( container, @@ -196,12 +194,10 @@ export class GenericContainer implements TestContainer { await startContainer(container); - const { dockerode, provider, host, hostIps } = await dockerClient(); + const { host, hostIps } = await dockerClient(); const inspectResult = await inspectContainer(container); const boundPorts = BoundPorts.fromInspectResult(hostIps, inspectResult).filter(this.opts.exposedPorts); - const waitStrategy = ( - this.waitStrategy ?? defaultWaitStrategy(host, dockerode, provider, container) - ).withStartupTimeout(this.startupTimeout); + const waitStrategy = (this.waitStrategy ?? Wait.forListeningPorts()).withStartupTimeout(this.startupTimeout); if (containerLog.enabled()) { (await containerLogs(container)) @@ -213,7 +209,7 @@ export class GenericContainer implements TestContainer { await this.containerStarting(inspectResult, false); } - await waitForContainer(container, waitStrategy, host, boundPorts); + await waitForContainer(container, waitStrategy, boundPorts); const startedContainer = new StartedGenericContainer( container, diff --git a/src/generic-container/started-generic-container.ts b/src/generic-container/started-generic-container.ts index e3da9e714..005d4e403 100644 --- a/src/generic-container/started-generic-container.ts +++ b/src/generic-container/started-generic-container.ts @@ -52,7 +52,7 @@ export class StartedGenericContainer implements StartedTestContainer { Array.from(this.boundPorts.iterator()).map((port) => port[0]) ); - await waitForContainer(this.container, this.waitStrategy, this.host, this.boundPorts, startTime); + await waitForContainer(this.container, this.waitStrategy, this.boundPorts, startTime); } private async stopContainer(options: Partial = {}): Promise { diff --git a/src/wait-for-container.ts b/src/wait-for-container.ts index e9b853495..af25bb74f 100644 --- a/src/wait-for-container.ts +++ b/src/wait-for-container.ts @@ -8,14 +8,13 @@ import { WaitStrategy } from "./wait-strategy/wait-strategy"; export const waitForContainer = async ( container: Dockerode.Container, waitStrategy: WaitStrategy, - host: string, boundPorts: BoundPorts, startTime?: Date ): Promise => { log.debug(`Waiting for container to be ready: ${container.id}`); try { - await waitStrategy.waitUntilReady(container, host, boundPorts, startTime); + await waitStrategy.waitUntilReady(container, boundPorts, startTime); log.info(`Container is ready: ${container.id}`); } catch (err) { log.error(`Container failed to be ready: ${container.id}: ${err}`); diff --git a/src/wait-strategy/default-wait-strategy.ts b/src/wait-strategy/default-wait-strategy.ts deleted file mode 100644 index 1bdcdec6b..000000000 --- a/src/wait-strategy/default-wait-strategy.ts +++ /dev/null @@ -1,11 +0,0 @@ -import Dockerode from "dockerode"; -import { HostPortWaitStrategy } from "./host-port-wait-strategy"; -import { HostPortCheck, InternalPortCheck } from "../port-check"; -import { Provider } from "../docker/docker-client"; - -export const defaultWaitStrategy = ( - host: string, - dockerode: Dockerode, - provider: Provider, - container: Dockerode.Container -) => new HostPortWaitStrategy(new HostPortCheck(host), new InternalPortCheck(dockerode, provider, container)); diff --git a/src/wait-strategy/host-port-wait-strategy.ts b/src/wait-strategy/host-port-wait-strategy.ts index 3505ff62d..7f9693273 100644 --- a/src/wait-strategy/host-port-wait-strategy.ts +++ b/src/wait-strategy/host-port-wait-strategy.ts @@ -1,31 +1,43 @@ -import { PortCheck } from "../port-check"; +import { HostPortCheck, InternalPortCheck, PortCheck } from "../port-check"; import Dockerode from "dockerode"; import { BoundPorts } from "../bound-ports"; import { log } from "../logger"; import { IntervalRetryStrategy } from "../retry-strategy"; import { AbstractWaitStrategy } from "./wait-strategy"; +import { dockerClient } from "../docker/docker-client"; export class HostPortWaitStrategy extends AbstractWaitStrategy { - constructor(private readonly hostPortCheck: PortCheck, private readonly internalPortCheck: PortCheck) { - super(); - } + public async waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts): Promise { + const { dockerode, provider, host } = await dockerClient(); + const hostPortCheck = new HostPortCheck(host); + const internalPortCheck = new InternalPortCheck(dockerode, provider, container); - public async waitUntilReady(container: Dockerode.Container, host: string, boundPorts: BoundPorts): Promise { - await Promise.all([this.waitForHostPorts(container, boundPorts), this.waitForInternalPorts(container, boundPorts)]); + await Promise.all([ + this.waitForHostPorts(hostPortCheck, container, boundPorts), + this.waitForInternalPorts(internalPortCheck, container, boundPorts), + ]); } - private async waitForHostPorts(container: Dockerode.Container, boundPorts: BoundPorts): Promise { + private async waitForHostPorts( + portCheck: PortCheck, + container: Dockerode.Container, + boundPorts: BoundPorts + ): Promise { for (const [, hostPort] of boundPorts.iterator()) { log.debug(`Waiting for host port ${hostPort} for ${container.id}`); - await this.waitForPort(container, hostPort, this.hostPortCheck); + await this.waitForPort(container, hostPort, portCheck); log.debug(`Host port ${hostPort} ready for ${container.id}`); } } - private async waitForInternalPorts(container: Dockerode.Container, boundPorts: BoundPorts): Promise { + private async waitForInternalPorts( + portCheck: PortCheck, + container: Dockerode.Container, + boundPorts: BoundPorts + ): Promise { for (const [internalPort] of boundPorts.iterator()) { log.debug(`Waiting for internal port ${internalPort} for ${container.id}`); - await this.waitForPort(container, internalPort, this.internalPortCheck); + await this.waitForPort(container, internalPort, portCheck); log.debug(`Internal port ${internalPort} ready for ${container.id}`); } } diff --git a/src/wait-strategy/http-wait-strategy.ts b/src/wait-strategy/http-wait-strategy.ts index 1ba25161a..5bb44d405 100644 --- a/src/wait-strategy/http-wait-strategy.ts +++ b/src/wait-strategy/http-wait-strategy.ts @@ -4,6 +4,7 @@ import { AbstractWaitStrategy } from "./wait-strategy"; import { IntervalRetryStrategy } from "../retry-strategy"; import fetch, { Response } from "node-fetch"; import https, { Agent } from "https"; +import { dockerClient } from "../docker/docker-client"; export class HttpWaitStrategy extends AbstractWaitStrategy { private readonly path: string; @@ -68,7 +69,8 @@ export class HttpWaitStrategy extends AbstractWaitStrategy { return this; } - public async waitUntilReady(container: Dockerode.Container, host: string, boundPorts: BoundPorts): Promise { + public async waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts): Promise { + const { host } = await dockerClient(); await new IntervalRetryStrategy(this.readTimeout).retryUntil( async () => { try { diff --git a/src/wait-strategy/log-wait-strategy.ts b/src/wait-strategy/log-wait-strategy.ts index 1149f45dd..ed5ef268a 100644 --- a/src/wait-strategy/log-wait-strategy.ts +++ b/src/wait-strategy/log-wait-strategy.ts @@ -12,12 +12,7 @@ export class LogWaitStrategy extends AbstractWaitStrategy { super(); } - public async waitUntilReady( - container: Dockerode.Container, - host: string, - boundPorts: BoundPorts, - startTime?: Date - ): Promise { + public async waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts, startTime?: Date): Promise { log.debug(`Waiting for log message "${this.message}" for ${container.id}`); const stream = await containerLogs(container, { since: startTime }); diff --git a/src/wait-strategy/wait-strategy.ts b/src/wait-strategy/wait-strategy.ts index ebab042eb..93bb403a1 100644 --- a/src/wait-strategy/wait-strategy.ts +++ b/src/wait-strategy/wait-strategy.ts @@ -2,7 +2,7 @@ import { BoundPorts } from "../bound-ports"; import Dockerode from "dockerode"; export interface WaitStrategy { - waitUntilReady(container: Dockerode.Container, host: string, boundPorts: BoundPorts, startTime?: Date): Promise; + waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts, startTime?: Date): Promise; withStartupTimeout(startupTimeout: number): WaitStrategy; } @@ -12,7 +12,6 @@ export abstract class AbstractWaitStrategy implements WaitStrategy { public abstract waitUntilReady( container: Dockerode.Container, - host: string, boundPorts: BoundPorts, startTime?: Date ): Promise; diff --git a/src/wait-strategy/wait.ts b/src/wait-strategy/wait.ts index e4627b4b5..cdeda3d4b 100644 --- a/src/wait-strategy/wait.ts +++ b/src/wait-strategy/wait.ts @@ -3,8 +3,13 @@ import { HttpWaitStrategy } from "./http-wait-strategy"; import { HealthCheckWaitStrategy } from "./health-check-wait-strategy"; import { Log, LogWaitStrategy } from "./log-wait-strategy"; import { ShellWaitStrategy } from "./shell-wait-strategy"; +import { HostPortWaitStrategy } from "./host-port-wait-strategy"; export class Wait { + public static forListeningPorts(): WaitStrategy { + return new HostPortWaitStrategy(); + } + public static forLogMessage(message: Log | RegExp, times = 1): WaitStrategy { return new LogWaitStrategy(message, times); } From 15a7271bcf102d8905c99a7bf7981e76d72c990b Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 6 Apr 2023 17:59:55 +0100 Subject: [PATCH 02/35] Implement and use CompositeWaitStrategy --- .../docker-compose-environment.test.ts | 33 ++++++++++++------- src/modules/arangodb/arangodb-container.ts | 2 +- .../elasticsearch/elasticsearch-container.ts | 2 +- src/modules/mongodb/mongodb-container.ts | 2 +- src/modules/nats/nats-container.ts | 32 ++++++++---------- src/modules/neo4j/neo4j-container.ts | 2 +- .../postgresql/postgresql-container.ts | 7 +++- src/reaper.ts | 2 +- .../composite-wait-strategy.test.ts | 3 ++ src/wait-strategy/composite-wait-strategy.ts | 15 +++++++++ .../health-check-wait-strategy.test.ts | 6 ++-- .../host-port-wait-strategy.test.ts | 2 +- src/wait-strategy/log-wait-strategy.test.ts | 4 +-- src/wait-strategy/wait.ts | 5 +++ 14 files changed, 75 insertions(+), 42 deletions(-) create mode 100644 src/wait-strategy/composite-wait-strategy.test.ts create mode 100644 src/wait-strategy/composite-wait-strategy.ts diff --git a/src/docker-compose-environment/docker-compose-environment.test.ts b/src/docker-compose-environment/docker-compose-environment.test.ts index 9448faf1c..97e4d4b72 100644 --- a/src/docker-compose-environment/docker-compose-environment.test.ts +++ b/src/docker-compose-environment/docker-compose-environment.test.ts @@ -84,8 +84,14 @@ describe("DockerComposeEnvironment", () => { it("should support log message wait strategy", async () => { const startedEnvironment = await new DockerComposeEnvironment(fixtures, "docker-compose.yml") - .withWaitStrategy(await composeContainerName("container"), Wait.forLogMessage("Listening on port 8080")) - .withWaitStrategy(await composeContainerName("another_container"), Wait.forLogMessage("Listening on port 8080")) + .withWaitStrategy( + await composeContainerName("container"), + Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage("Listening on port 8080")]) + ) + .withWaitStrategy( + await composeContainerName("another_container"), + Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage("Listening on port 8080")]) + ) .up(); await Promise.all( @@ -108,6 +114,19 @@ describe("DockerComposeEnvironment", () => { expect(await getRunningContainerNames()).not.toContain("custom_container_name"); }); + it("should support health check wait strategy", async () => { + const startedEnvironment = await new DockerComposeEnvironment(fixtures, "docker-compose-with-healthcheck.yml") + .withWaitStrategy( + await composeContainerName("container"), + Wait.forAll([Wait.forListeningPorts(), Wait.forHealthCheck()]) + ) + .up(); + + await checkEnvironmentContainerIsHealthy(startedEnvironment, await composeContainerName("container")); + + await startedEnvironment.down(); + }); + it("should stop the container when the health check wait strategy times out", async () => { await expect( new DockerComposeEnvironment(fixtures, "docker-compose-with-healthcheck.yml") @@ -119,16 +138,6 @@ describe("DockerComposeEnvironment", () => { expect(await getRunningContainerNames()).not.toContain("container_1"); }); - it("should support health check wait strategy", async () => { - const startedEnvironment = await new DockerComposeEnvironment(fixtures, "docker-compose-with-healthcheck.yml") - .withWaitStrategy(await composeContainerName("container"), Wait.forHealthCheck()) - .up(); - - await checkEnvironmentContainerIsHealthy(startedEnvironment, await composeContainerName("container")); - - await startedEnvironment.down(); - }); - it("should remove volumes when downing an environment", async () => { const environment = await new DockerComposeEnvironment(fixtures, "docker-compose-with-volume.yml").up(); diff --git a/src/modules/arangodb/arangodb-container.ts b/src/modules/arangodb/arangodb-container.ts index 9e82d3bee..7373b7d73 100755 --- a/src/modules/arangodb/arangodb-container.ts +++ b/src/modules/arangodb/arangodb-container.ts @@ -17,7 +17,7 @@ export class ArangoDBContainer extends GenericContainer { public override async start(): Promise { this.withExposedPorts(...(this.hasExposedPorts ? this.opts.exposedPorts : [ARANGODB_PORT])) - .withWaitStrategy(Wait.forLogMessage("Have fun!")) + .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage("Have fun!")])) .withEnvironment({ ARANGO_ROOT_PASSWORD: this.password }) .withStartupTimeout(120_000); diff --git a/src/modules/elasticsearch/elasticsearch-container.ts b/src/modules/elasticsearch/elasticsearch-container.ts index 2fa4112d5..ed6400e9f 100644 --- a/src/modules/elasticsearch/elasticsearch-container.ts +++ b/src/modules/elasticsearch/elasticsearch-container.ts @@ -13,7 +13,7 @@ export class ElasticsearchContainer extends GenericContainer { .withEnvironment({ "discovery.type": "single-node" }) .withCopyContentToContainer([ { - content: "-Xms2G\n-Xmx2G\n", + content: "-Xmx2G\n", target: "/usr/share/elasticsearch/config/jvm.options.d/elasticsearch-default-memory-vm.options", }, ]) diff --git a/src/modules/mongodb/mongodb-container.ts b/src/modules/mongodb/mongodb-container.ts index 76f4ffc5a..ae8907f5f 100644 --- a/src/modules/mongodb/mongodb-container.ts +++ b/src/modules/mongodb/mongodb-container.ts @@ -17,7 +17,7 @@ export class MongoDBContainer extends GenericContainer { public override async start(): Promise { this.withExposedPorts(MONGODB_PORT) .withCommand(["--replSet", "rs0"]) - .withWaitStrategy(Wait.forLogMessage(/.*waiting for connections.*/i)) + .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage(/.*waiting for connections.*/i)])) .withStartupTimeout(120_000); return new StartedMongoDBContainer(await super.start()); } diff --git a/src/modules/nats/nats-container.ts b/src/modules/nats/nats-container.ts index ada4641b4..e15e9a272 100755 --- a/src/modules/nats/nats-container.ts +++ b/src/modules/nats/nats-container.ts @@ -35,18 +35,6 @@ export class NatsContainer extends GenericContainer { return this; } - private static ensureDashInFrontOfArgumentName(name: string): string { - if (name.startsWith("--") || name.startsWith("-")) { - return name; - } - - if (name.length == 1) { - return "-" + name; - } else { - return "--" + name; - } - } - public override async start(): Promise { function buildCmdsFromArgs(args: { [p: string]: string }): string[] { const result: string[] = []; @@ -60,17 +48,25 @@ export class NatsContainer extends GenericContainer { } this.withCommand(buildCmdsFromArgs(this.args)) - .withExposedPorts( - ...(this.hasExposedPorts - ? this.opts.exposedPorts - : [CLIENT_PORT, ROUTING_PORT_FOR_CLUSTERING, HTTP_MANAGEMENT_PORT]) - ) - .withWaitStrategy(Wait.forLogMessage(/.*Server is ready.*/)) + .withExposedPorts(...(this.hasExposedPorts ? this.opts.exposedPorts : [CLIENT_PORT])) + .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage(/.*Server is ready.*/)])) .withStartupTimeout(120_000); return new StartedNatsContainer(await super.start(), this.getUser(), this.getPass()); } + private static ensureDashInFrontOfArgumentName(name: string): string { + if (name.startsWith("--") || name.startsWith("-")) { + return name; + } + + if (name.length == 1) { + return "-" + name; + } else { + return "--" + name; + } + } + private getUser(): string { return this.args[USER_ARGUMENT_KEY]; } diff --git a/src/modules/neo4j/neo4j-container.ts b/src/modules/neo4j/neo4j-container.ts index 0b182a42b..f09444b1c 100755 --- a/src/modules/neo4j/neo4j-container.ts +++ b/src/modules/neo4j/neo4j-container.ts @@ -32,7 +32,7 @@ export class Neo4jContainer extends GenericContainer { public override async start(): Promise { this.withExposedPorts(...(this.hasExposedPorts ? this.opts.exposedPorts : [BOLT_PORT, HTTP_PORT])) - .withWaitStrategy(Wait.forLogMessage("Started.")) + .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage("Started.")])) .withEnvironment({ NEO4J_AUTH: `${USERNAME}/${this.password}` }) .withStartupTimeout(120_000); diff --git a/src/modules/postgresql/postgresql-container.ts b/src/modules/postgresql/postgresql-container.ts index f51817840..f8f80ba26 100755 --- a/src/modules/postgresql/postgresql-container.ts +++ b/src/modules/postgresql/postgresql-container.ts @@ -36,7 +36,12 @@ export class PostgreSqlContainer extends GenericContainer { POSTGRES_USER: this.username, POSTGRES_PASSWORD: this.password, }) - .withWaitStrategy(Wait.forLogMessage(/.*database system is ready to accept connections.*/, 2)) + .withWaitStrategy( + Wait.forAll([ + Wait.forListeningPorts(), + Wait.forLogMessage(/.*database system is ready to accept connections.*/, 2), + ]) + ) .withStartupTimeout(120_000); return new StartedPostgreSqlContainer(await super.start(), this.database, this.username, this.password); diff --git a/src/reaper.ts b/src/reaper.ts index 9bc357eb2..0b47ac8f1 100644 --- a/src/reaper.ts +++ b/src/reaper.ts @@ -100,7 +100,7 @@ export class ReaperInstance { target: "/var/run/docker.sock", }, ]) - .withWaitStrategy(Wait.forLogMessage(/.+ Started!/)); + .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage(/.+ Started!/)])); if (this.isPrivileged()) { container.withPrivilegedMode(); diff --git a/src/wait-strategy/composite-wait-strategy.test.ts b/src/wait-strategy/composite-wait-strategy.test.ts new file mode 100644 index 000000000..6837a8081 --- /dev/null +++ b/src/wait-strategy/composite-wait-strategy.test.ts @@ -0,0 +1,3 @@ +test("passes", () => { + // todo +}); diff --git a/src/wait-strategy/composite-wait-strategy.ts b/src/wait-strategy/composite-wait-strategy.ts new file mode 100644 index 000000000..5e4bac044 --- /dev/null +++ b/src/wait-strategy/composite-wait-strategy.ts @@ -0,0 +1,15 @@ +import { AbstractWaitStrategy, WaitStrategy } from "./wait-strategy"; +import Dockerode from "dockerode"; +import { BoundPorts } from "../bound-ports"; + +export class CompositeWaitStrategy extends AbstractWaitStrategy { + constructor(private readonly waitStrategies: WaitStrategy[]) { + super(); + } + + public async waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts, startTime?: Date): Promise { + await Promise.all( + this.waitStrategies.map((waitStrategy) => waitStrategy.waitUntilReady(container, boundPorts, startTime)) + ); + } +} diff --git a/src/wait-strategy/health-check-wait-strategy.test.ts b/src/wait-strategy/health-check-wait-strategy.test.ts index 47fa4b5e8..ff4c7411e 100644 --- a/src/wait-strategy/health-check-wait-strategy.test.ts +++ b/src/wait-strategy/health-check-wait-strategy.test.ts @@ -14,7 +14,7 @@ describe("HealthCheckWaitStrategy", () => { const customGenericContainer = await GenericContainer.fromDockerfile(context).build(); const container = await customGenericContainer .withExposedPorts(8080) - .withWaitStrategy(Wait.forHealthCheck()) + .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forHealthCheck()])) .start(); await checkContainerIsHealthy(container); @@ -32,7 +32,7 @@ describe("HealthCheckWaitStrategy", () => { retries: 5, startPeriod: 1000, }) - .withWaitStrategy(Wait.forHealthCheck()) + .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forHealthCheck()])) .start(); await checkContainerIsHealthy(container); @@ -84,7 +84,7 @@ describe("HealthCheckWaitStrategy", () => { retries: 5, startPeriod: 1000, }) - .withWaitStrategy(Wait.forHealthCheck()) + .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forHealthCheck()])) .start(); await checkContainerIsHealthy(container); diff --git a/src/wait-strategy/host-port-wait-strategy.test.ts b/src/wait-strategy/host-port-wait-strategy.test.ts index 9e7bbada2..5f78088a1 100644 --- a/src/wait-strategy/host-port-wait-strategy.test.ts +++ b/src/wait-strategy/host-port-wait-strategy.test.ts @@ -22,7 +22,7 @@ describe("HostPortWaitStrategy", () => { .withExposedPorts(8081) .withStartupTimeout(0) .start() - ).rejects.toThrowError("Port 8081 not bound after 0ms"); + ).rejects.toThrowError(/Port \d+ not bound after 0ms/); expect(await getRunningContainerNames()).not.toContain(containerName); }); diff --git a/src/wait-strategy/log-wait-strategy.test.ts b/src/wait-strategy/log-wait-strategy.test.ts index 2b6be720c..ea5ee33e0 100644 --- a/src/wait-strategy/log-wait-strategy.test.ts +++ b/src/wait-strategy/log-wait-strategy.test.ts @@ -9,7 +9,7 @@ describe("LogWaitStrategy", () => { it("should wait for log", async () => { const container = await new GenericContainer("cristianrgreco/testcontainer:1.1.14") .withExposedPorts(8080) - .withWaitStrategy(Wait.forLogMessage("Listening on port 8080")) + .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage("Listening on port 8080")])) .start(); await checkContainerIsHealthy(container); @@ -20,7 +20,7 @@ describe("LogWaitStrategy", () => { it("should wait for log with regex", async () => { const container = await new GenericContainer("cristianrgreco/testcontainer:1.1.14") .withExposedPorts(8080) - .withWaitStrategy(Wait.forLogMessage(/Listening on port \d+/)) + .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage(/Listening on port \d+/)])) .start(); await checkContainerIsHealthy(container); diff --git a/src/wait-strategy/wait.ts b/src/wait-strategy/wait.ts index cdeda3d4b..a17df089e 100644 --- a/src/wait-strategy/wait.ts +++ b/src/wait-strategy/wait.ts @@ -4,8 +4,13 @@ import { HealthCheckWaitStrategy } from "./health-check-wait-strategy"; import { Log, LogWaitStrategy } from "./log-wait-strategy"; import { ShellWaitStrategy } from "./shell-wait-strategy"; import { HostPortWaitStrategy } from "./host-port-wait-strategy"; +import { CompositeWaitStrategy } from "./composite-wait-strategy"; export class Wait { + public static forAll(waitStrategies: WaitStrategy[]): WaitStrategy { + return new CompositeWaitStrategy(waitStrategies); + } + public static forListeningPorts(): WaitStrategy { return new HostPortWaitStrategy(); } From c05ae8cc8e1012f86c73ba0b88facdb93208d3b7 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 10:45:09 +0100 Subject: [PATCH 03/35] Add colima workflow, disable others --- .github/workflows/test.yml | 191 +++++++++++++++++++++---------------- 1 file changed, 110 insertions(+), 81 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1cc0a03c8..250e662f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,26 +13,88 @@ concurrency: cancel-in-progress: true jobs: - docker: - runs-on: ${{ matrix.os-version }} - strategy: - fail-fast: false - matrix: - os-version: [ ubuntu-latest ] - node-version: [ 14.x, 16.x, 18.x ] - steps: - - name: Code checkout - uses: actions/checkout@v3 - - name: Install NodeJS ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - name: Install dependencies - run: npm ci --omit=optional - - name: Run tests - run: npm run test:ci +# docker: +# runs-on: ${{ matrix.os-version }} +# strategy: +# fail-fast: false +# matrix: +# os-version: [ ubuntu-latest ] +# node-version: [ 14.x, 16.x, 18.x ] +# steps: +# - name: Code checkout +# uses: actions/checkout@v3 +# - name: Install NodeJS ${{ matrix.node-version }} +# uses: actions/setup-node@v3 +# with: +# node-version: ${{ matrix.node-version }} +# - name: Install dependencies +# run: npm ci --omit=optional +# - name: Run tests +# run: npm run test:ci +# +# docker-rootless: +# runs-on: ${{ matrix.os-version }} +# strategy: +# fail-fast: false +# matrix: +# os-version: [ ubuntu-latest ] +# node-version: [ 14.x, 16.x, 18.x ] +# steps: +# - name: Code checkout +# uses: actions/checkout@v3 +# - name: Setup rootless Docker +# uses: ScribeMD/rootless-docker@0.2.2 +# - name: Remove Docket root socket +# run: sudo rm -rf /var/run/docker.sock +# - name: Install NodeJS +# uses: actions/setup-node@v3 +# with: +# node-version: ${{ matrix.node-version }} +# - name: Install dependencies +# run: npm ci --omit=optional +# - name: Run tests +# run: npm run test:ci +# env: +# CI_ROOTLESS: true +# +# podman: +# runs-on: ${{ matrix.os-version }} +# strategy: +# fail-fast: false +# matrix: +# os-version: [ ubuntu-latest ] +# node-version: [ 14.x, 16.x, 18.x ] +# steps: +# - name: Code checkout +# uses: actions/checkout@v3 +# - name: Setup Podman +# run: | +# curl -fsSL https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/Debian_Testing/Release.key \ +# | gpg --dearmor \ +# | sudo tee /etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg > /dev/null +# echo \ +# "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg]\ +# https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/Debian_Testing/ /" \ +# | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list > /dev/null +# sudo apt-get update +# sudo apt-get -y install podman +# systemctl enable --now --user podman podman.socket +# podman info +# - name: Set environment +# run: echo "DOCKER_HOST=unix://${XDG_RUNTIME_DIR}/podman/podman.sock" >> $GITHUB_ENV +# - name: Install NodeJS ${{ matrix.node-version }} +# uses: actions/setup-node@v3 +# with: +# node-version: ${{ matrix.node-version }} +# - name: Install dependencies +# run: npm ci --omit=optional +# - name: Run tests +# run: npm run test:ci +# env: +# CI_ROOTLESS: true +# CI_PODMAN: true - docker-rootless: + colima: runs-on: ${{ matrix.os-version }} strategy: fail-fast: false @@ -42,46 +104,16 @@ jobs: steps: - name: Code checkout uses: actions/checkout@v3 - - name: Setup rootless Docker - uses: ScribeMD/rootless-docker@0.2.2 - - name: Remove Docket root socket - run: sudo rm -rf /var/run/docker.sock - - name: Install NodeJS - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - name: Install dependencies - run: npm ci --omit=optional - - name: Run tests - run: npm run test:ci - env: - CI_ROOTLESS: true - - podman: - runs-on: ${{ matrix.os-version }} - strategy: - fail-fast: false - matrix: - os-version: [ ubuntu-latest ] - node-version: [ 14.x, 16.x, 18.x ] - steps: - - name: Code checkout - uses: actions/checkout@v3 - - name: Setup Podman + - name: Setup Colima run: | - curl -fsSL https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/Debian_Testing/Release.key \ - | gpg --dearmor \ - | sudo tee /etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg > /dev/null - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg]\ - https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/Debian_Testing/ /" \ - | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list > /dev/null - sudo apt-get update - sudo apt-get -y install podman - systemctl enable --now --user podman podman.socket - podman info + curl -L https://nixos.org/nix/install | sh -s -- --daemon + nix-env -iA nixpkgs.colima + colima start --runtime docker + colima status - name: Set environment - run: echo "DOCKER_HOST=unix://${XDG_RUNTIME_DIR}/podman/podman.sock" >> $GITHUB_ENV + run: | + echo "DOCKER_HOST=unix://${HOME}/.colima/default/docker.sock" >> $GITHUB_ENV + echo "TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=/var/run/docker.sock" >> $GITHUB_ENV - name: Install NodeJS ${{ matrix.node-version }} uses: actions/setup-node@v3 with: @@ -90,28 +122,25 @@ jobs: run: npm ci --omit=optional - name: Run tests run: npm run test:ci - env: - CI_ROOTLESS: true - CI_PODMAN: true - smoke-test: - runs-on: ${{ matrix.os-version }} - strategy: - matrix: - os-version: [ ubuntu-latest ] - node-version: [ 14.x, 16.x, 18.x ] - steps: - - name: Code checkout - uses: actions/checkout@v3 - - name: Install NodeJS ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - name: Install dependencies - run: npm ci --omit=optional - - name: Remove dev dependencies - run: npm prune --omit=dev - - name: Run CommonJS module smoke test - run: node smoke-test.js - - name: Run ES module smoke test - run: node smoke-test.mjs \ No newline at end of file +# smoke-test: +# runs-on: ${{ matrix.os-version }} +# strategy: +# matrix: +# os-version: [ ubuntu-latest ] +# node-version: [ 14.x, 16.x, 18.x ] +# steps: +# - name: Code checkout +# uses: actions/checkout@v3 +# - name: Install NodeJS ${{ matrix.node-version }} +# uses: actions/setup-node@v3 +# with: +# node-version: ${{ matrix.node-version }} +# - name: Install dependencies +# run: npm ci --omit=optional +# - name: Remove dev dependencies +# run: npm prune --omit=dev +# - name: Run CommonJS module smoke test +# run: node smoke-test.js +# - name: Run ES module smoke test +# run: node smoke-test.mjs \ No newline at end of file From ef16dd73016d84c6a742ce5dbfe21a7c517308c3 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 10:47:00 +0100 Subject: [PATCH 04/35] Enable other jobs --- .github/workflows/test.yml | 202 ++++++++++++++++++------------------- 1 file changed, 101 insertions(+), 101 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 250e662f2..a30931454 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,86 +13,86 @@ concurrency: cancel-in-progress: true jobs: -# docker: -# runs-on: ${{ matrix.os-version }} -# strategy: -# fail-fast: false -# matrix: -# os-version: [ ubuntu-latest ] -# node-version: [ 14.x, 16.x, 18.x ] -# steps: -# - name: Code checkout -# uses: actions/checkout@v3 -# - name: Install NodeJS ${{ matrix.node-version }} -# uses: actions/setup-node@v3 -# with: -# node-version: ${{ matrix.node-version }} -# - name: Install dependencies -# run: npm ci --omit=optional -# - name: Run tests -# run: npm run test:ci -# -# docker-rootless: -# runs-on: ${{ matrix.os-version }} -# strategy: -# fail-fast: false -# matrix: -# os-version: [ ubuntu-latest ] -# node-version: [ 14.x, 16.x, 18.x ] -# steps: -# - name: Code checkout -# uses: actions/checkout@v3 -# - name: Setup rootless Docker -# uses: ScribeMD/rootless-docker@0.2.2 -# - name: Remove Docket root socket -# run: sudo rm -rf /var/run/docker.sock -# - name: Install NodeJS -# uses: actions/setup-node@v3 -# with: -# node-version: ${{ matrix.node-version }} -# - name: Install dependencies -# run: npm ci --omit=optional -# - name: Run tests -# run: npm run test:ci -# env: -# CI_ROOTLESS: true -# -# podman: -# runs-on: ${{ matrix.os-version }} -# strategy: -# fail-fast: false -# matrix: -# os-version: [ ubuntu-latest ] -# node-version: [ 14.x, 16.x, 18.x ] -# steps: -# - name: Code checkout -# uses: actions/checkout@v3 -# - name: Setup Podman -# run: | -# curl -fsSL https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/Debian_Testing/Release.key \ -# | gpg --dearmor \ -# | sudo tee /etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg > /dev/null -# echo \ -# "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg]\ -# https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/Debian_Testing/ /" \ -# | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list > /dev/null -# sudo apt-get update -# sudo apt-get -y install podman -# systemctl enable --now --user podman podman.socket -# podman info -# - name: Set environment -# run: echo "DOCKER_HOST=unix://${XDG_RUNTIME_DIR}/podman/podman.sock" >> $GITHUB_ENV -# - name: Install NodeJS ${{ matrix.node-version }} -# uses: actions/setup-node@v3 -# with: -# node-version: ${{ matrix.node-version }} -# - name: Install dependencies -# run: npm ci --omit=optional -# - name: Run tests -# run: npm run test:ci -# env: -# CI_ROOTLESS: true -# CI_PODMAN: true + docker: + runs-on: ${{ matrix.os-version }} + strategy: + fail-fast: false + matrix: + os-version: [ ubuntu-latest ] + node-version: [ 14.x, 16.x, 18.x ] + steps: + - name: Code checkout + uses: actions/checkout@v3 + - name: Install NodeJS ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: Install dependencies + run: npm ci --omit=optional + - name: Run tests + run: npm run test:ci + + docker-rootless: + runs-on: ${{ matrix.os-version }} + strategy: + fail-fast: false + matrix: + os-version: [ ubuntu-latest ] + node-version: [ 14.x, 16.x, 18.x ] + steps: + - name: Code checkout + uses: actions/checkout@v3 + - name: Setup rootless Docker + uses: ScribeMD/rootless-docker@0.2.2 + - name: Remove Docket root socket + run: sudo rm -rf /var/run/docker.sock + - name: Install NodeJS + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: Install dependencies + run: npm ci --omit=optional + - name: Run tests + run: npm run test:ci + env: + CI_ROOTLESS: true + + podman: + runs-on: ${{ matrix.os-version }} + strategy: + fail-fast: false + matrix: + os-version: [ ubuntu-latest ] + node-version: [ 14.x, 16.x, 18.x ] + steps: + - name: Code checkout + uses: actions/checkout@v3 + - name: Setup Podman + run: | + curl -fsSL https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/Debian_Testing/Release.key \ + | gpg --dearmor \ + | sudo tee /etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg > /dev/null + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg]\ + https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/Debian_Testing/ /" \ + | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list > /dev/null + sudo apt-get update + sudo apt-get -y install podman + systemctl enable --now --user podman podman.socket + podman info + - name: Set environment + run: echo "DOCKER_HOST=unix://${XDG_RUNTIME_DIR}/podman/podman.sock" >> $GITHUB_ENV + - name: Install NodeJS ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: Install dependencies + run: npm ci --omit=optional + - name: Run tests + run: npm run test:ci + env: + CI_ROOTLESS: true + CI_PODMAN: true colima: runs-on: ${{ matrix.os-version }} @@ -123,24 +123,24 @@ jobs: - name: Run tests run: npm run test:ci -# smoke-test: -# runs-on: ${{ matrix.os-version }} -# strategy: -# matrix: -# os-version: [ ubuntu-latest ] -# node-version: [ 14.x, 16.x, 18.x ] -# steps: -# - name: Code checkout -# uses: actions/checkout@v3 -# - name: Install NodeJS ${{ matrix.node-version }} -# uses: actions/setup-node@v3 -# with: -# node-version: ${{ matrix.node-version }} -# - name: Install dependencies -# run: npm ci --omit=optional -# - name: Remove dev dependencies -# run: npm prune --omit=dev -# - name: Run CommonJS module smoke test -# run: node smoke-test.js -# - name: Run ES module smoke test -# run: node smoke-test.mjs \ No newline at end of file + smoke-test: + runs-on: ${{ matrix.os-version }} + strategy: + matrix: + os-version: [ ubuntu-latest ] + node-version: [ 14.x, 16.x, 18.x ] + steps: + - name: Code checkout + uses: actions/checkout@v3 + - name: Install NodeJS ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: Install dependencies + run: npm ci --omit=optional + - name: Remove dev dependencies + run: npm prune --omit=dev + - name: Run CommonJS module smoke test + run: node smoke-test.js + - name: Run ES module smoke test + run: node smoke-test.mjs \ No newline at end of file From c38135a204ba6ab54e41bd7606bb22f0c81210d2 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 10:54:27 +0100 Subject: [PATCH 05/35] Update colima workflow --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a30931454..496b53325 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,6 +107,7 @@ jobs: - name: Setup Colima run: | curl -L https://nixos.org/nix/install | sh -s -- --daemon + source /etc/bash.bashrc nix-env -iA nixpkgs.colima colima start --runtime docker colima status From ab1931dd485ad2c44603bb0c946037bfdb5e25f5 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 10:58:39 +0100 Subject: [PATCH 06/35] Update workflow --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 496b53325..a121e34ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,10 +104,10 @@ jobs: steps: - name: Code checkout uses: actions/checkout@v3 + - name: Install nix + run: curl -L https://nixos.org/nix/install | sh -s -- --daemon - name: Setup Colima run: | - curl -L https://nixos.org/nix/install | sh -s -- --daemon - source /etc/bash.bashrc nix-env -iA nixpkgs.colima colima start --runtime docker colima status From 159fba4f943f8e909bdda5199ff3e166cd587e0b Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 11:02:15 +0100 Subject: [PATCH 07/35] Update workflow --- .github/workflows/test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a121e34ac..4943992f6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,11 +104,10 @@ jobs: steps: - name: Code checkout uses: actions/checkout@v3 - - name: Install nix - run: curl -L https://nixos.org/nix/install | sh -s -- --daemon - name: Setup Colima run: | - nix-env -iA nixpkgs.colima + curl -L https://nixos.org/nix/install | sh -s -- --daemon + bash -lc "nix-env -iA nixpkgs.colima" colima start --runtime docker colima status - name: Set environment From 0447b1d150dcc55abe3c21a75c08a626a603ad92 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 11:05:29 +0100 Subject: [PATCH 08/35] Update workflow --- .github/workflows/test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4943992f6..6b60c49c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,9 +107,11 @@ jobs: - name: Setup Colima run: | curl -L https://nixos.org/nix/install | sh -s -- --daemon - bash -lc "nix-env -iA nixpkgs.colima" - colima start --runtime docker - colima status + bash -lc 'cat <> $GITHUB_ENV From 2e789614a2a097d94ae02257ab48de622c1c5adb Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 11:07:14 +0100 Subject: [PATCH 09/35] Update workflow --- .github/workflows/test.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6b60c49c0..c86c7e79d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,11 +107,9 @@ jobs: - name: Setup Colima run: | curl -L https://nixos.org/nix/install | sh -s -- --daemon - bash -lc 'cat <> $GITHUB_ENV From 4f5cbe197eb28553a4a99291d14e37b2d9082e9a Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 11:14:02 +0100 Subject: [PATCH 10/35] Update workflow --- .github/workflows/test.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c86c7e79d..d8765af74 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,17 +99,15 @@ jobs: strategy: fail-fast: false matrix: - os-version: [ ubuntu-latest ] + os-version: [ macos-latest ] node-version: [ 14.x, 16.x, 18.x ] steps: - name: Code checkout uses: actions/checkout@v3 - - name: Setup Colima + - name: Start Colima run: | - curl -L https://nixos.org/nix/install | sh -s -- --daemon - bash -lc "nix-env -iA nixpkgs.colima" - bash -lc "colima start --runtime docker" - bash -lc "colima status" + colima start --runtime docker + colima status - name: Set environment run: | echo "DOCKER_HOST=unix://${HOME}/.colima/default/docker.sock" >> $GITHUB_ENV From d4bb41e0c7cb73eeb7bb896f057aad6114f477c4 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 11:15:28 +0100 Subject: [PATCH 11/35] Update workflow --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d8765af74..35267f81e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,6 +106,7 @@ jobs: uses: actions/checkout@v3 - name: Start Colima run: | + brew install docker colima start --runtime docker colima status - name: Set environment From efc660d5a9da3eabe1da274da1658cc5a6a7b930 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 11:52:49 +0100 Subject: [PATCH 12/35] Update workflow --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35267f81e..25f676a17 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,7 +106,7 @@ jobs: uses: actions/checkout@v3 - name: Start Colima run: | - brew install docker + brew install docker docker-compose colima start --runtime docker colima status - name: Set environment From ed49fbc9dae183ff30858866647268245e2e19c9 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 11:55:46 +0100 Subject: [PATCH 13/35] Update workflow --- src/modules/nats/nats-container.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/modules/nats/nats-container.ts b/src/modules/nats/nats-container.ts index e15e9a272..da532600b 100755 --- a/src/modules/nats/nats-container.ts +++ b/src/modules/nats/nats-container.ts @@ -4,8 +4,6 @@ import { AbstractStartedContainer } from "../abstract-started-container"; import { Wait } from "../../wait-strategy/wait"; const CLIENT_PORT = 4222; -const ROUTING_PORT_FOR_CLUSTERING = 6222; -const HTTP_MANAGEMENT_PORT = 8222; const USER_ARGUMENT_KEY = "--user"; const PASS_ARGUMENT_KEY = "--pass"; From bf930504f9b3c3d7127581c6773b194a5fe5d9e1 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 11 Apr 2023 12:01:25 +0100 Subject: [PATCH 14/35] Update workflow --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 25f676a17..740d3fd80 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,7 +107,7 @@ jobs: - name: Start Colima run: | brew install docker docker-compose - colima start --runtime docker + colima start --cpu 3 --memory 10 --disk 10 --runtime docker # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources colima status - name: Set environment run: | From 265b608fc5794aa52703bcf00fbdff6e8ed71b0f Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Wed, 12 Apr 2023 09:15:22 +0100 Subject: [PATCH 15/35] Fix some flakiness --- fixtures/docker-compose/docker-compose-with-healthcheck.yml | 2 +- src/modules/nats/nats-container.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fixtures/docker-compose/docker-compose-with-healthcheck.yml b/fixtures/docker-compose/docker-compose-with-healthcheck.yml index b4a1fd16e..7d34f2605 100644 --- a/fixtures/docker-compose/docker-compose-with-healthcheck.yml +++ b/fixtures/docker-compose/docker-compose-with-healthcheck.yml @@ -10,4 +10,4 @@ services: interval: 1s timeout: 3s retries: 10 - start_period: 0s + start_period: 1s diff --git a/src/modules/nats/nats-container.test.ts b/src/modules/nats/nats-container.test.ts index 82a5737d9..927eb24aa 100644 --- a/src/modules/nats/nats-container.test.ts +++ b/src/modules/nats/nats-container.test.ts @@ -75,6 +75,6 @@ describe("NatsContainer", () => { await connect(container.getConnectionOptions()); } - await expect(outputVersionAndExit()).rejects.toThrow("Log stream ended"); + await expect(outputVersionAndExit()).rejects.toThrowError(); }); }); From fd76837332cfab8b78a889f489655dcf9635bf14 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Wed, 12 Apr 2023 09:45:07 +0100 Subject: [PATCH 16/35] CompositeWaitStrategy defaults to max startupTimeout --- .../composite-wait-strategy.test.ts | 60 ++++++++++++++++++- src/wait-strategy/composite-wait-strategy.ts | 5 ++ src/wait-strategy/wait-strategy.ts | 8 ++- 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/src/wait-strategy/composite-wait-strategy.test.ts b/src/wait-strategy/composite-wait-strategy.test.ts index 6837a8081..dccb49f8f 100644 --- a/src/wait-strategy/composite-wait-strategy.test.ts +++ b/src/wait-strategy/composite-wait-strategy.test.ts @@ -1,3 +1,59 @@ -test("passes", () => { - // todo +import { CompositeWaitStrategy } from "./composite-wait-strategy"; +import { AbstractWaitStrategy } from "./wait-strategy"; +import { BoundPorts } from "../bound-ports"; +import Dockerode from "dockerode"; + +const fakeContainer = {} as Dockerode.Container; +const fakeBoundPorts = {} as BoundPorts; +const fakeStartTime = {} as Date; + +test("should resolve when all wait strategies resolve", async () => { + const waitStrategy = new CompositeWaitStrategy([ + new FakeWaitStrategy({ resolves: true }), + new FakeWaitStrategy({ resolves: true }), + ]); + + await expect(waitStrategy.waitUntilReady(fakeContainer, fakeBoundPorts, fakeStartTime)).resolves.toBeUndefined(); }); + +test("should reject when any wait strategy rejects", async () => { + const error = new Error("FAILED"); + const waitStrategy = new CompositeWaitStrategy([ + new FakeWaitStrategy({ resolves: true }), + new FakeWaitStrategy({ resolves: false, error }), + ]); + + await expect(waitStrategy.waitUntilReady(fakeContainer, fakeBoundPorts, fakeStartTime)).rejects.toThrowError(error); +}); + +test("should set the startup timeout to the maximum of all wait strategies", async () => { + const waitStrategy = new CompositeWaitStrategy([ + new FakeWaitStrategy({ resolves: true }).withStartupTimeout(1000), + new FakeWaitStrategy({ resolves: true }).withStartupTimeout(2000), + ]); + + expect(waitStrategy.getStartupTimeout()).toEqual(2000); +}); + +test("should override startup timeouts when once is provided", async () => { + const waitStrategy = new CompositeWaitStrategy([ + new FakeWaitStrategy({ resolves: true }).withStartupTimeout(1000), + new FakeWaitStrategy({ resolves: true }).withStartupTimeout(2000), + ]).withStartupTimeout(3000); + + expect(waitStrategy.getStartupTimeout()).toEqual(3000); +}); + +class FakeWaitStrategy extends AbstractWaitStrategy { + constructor(private opts: { resolves: boolean; error?: Error }) { + super(); + } + + waitUntilReady(): Promise { + if (this.opts.resolves) { + return Promise.resolve(); + } else { + return Promise.reject(this.opts.error); + } + } +} diff --git a/src/wait-strategy/composite-wait-strategy.ts b/src/wait-strategy/composite-wait-strategy.ts index 5e4bac044..21746664c 100644 --- a/src/wait-strategy/composite-wait-strategy.ts +++ b/src/wait-strategy/composite-wait-strategy.ts @@ -5,6 +5,11 @@ import { BoundPorts } from "../bound-ports"; export class CompositeWaitStrategy extends AbstractWaitStrategy { constructor(private readonly waitStrategies: WaitStrategy[]) { super(); + this.withStartupTimeout(this.getMaxStartupTimeout()); + } + + private getMaxStartupTimeout(): number { + return Math.max(...this.waitStrategies.map((waitStrategy) => waitStrategy.getStartupTimeout())); } public async waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts, startTime?: Date): Promise { diff --git a/src/wait-strategy/wait-strategy.ts b/src/wait-strategy/wait-strategy.ts index 93bb403a1..a57103650 100644 --- a/src/wait-strategy/wait-strategy.ts +++ b/src/wait-strategy/wait-strategy.ts @@ -5,6 +5,8 @@ export interface WaitStrategy { waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts, startTime?: Date): Promise; withStartupTimeout(startupTimeout: number): WaitStrategy; + + getStartupTimeout(): number; } export abstract class AbstractWaitStrategy implements WaitStrategy { @@ -16,8 +18,12 @@ export abstract class AbstractWaitStrategy implements WaitStrategy { startTime?: Date ): Promise; - public withStartupTimeout(startupTimeout: number): WaitStrategy { + public withStartupTimeout(startupTimeout: number): this { this.startupTimeout = startupTimeout; return this; } + + public getStartupTimeout(): number { + return this.startupTimeout; + } } From 524584ee4587e34e5e92ca2f9af885c61fb1d6b9 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Wed, 12 Apr 2023 10:42:06 +0100 Subject: [PATCH 17/35] CompositeWaitStrategy fixes --- .../composite-wait-strategy.test.ts | 27 ++++++++++++---- src/wait-strategy/composite-wait-strategy.ts | 32 +++++++++++++++---- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/wait-strategy/composite-wait-strategy.test.ts b/src/wait-strategy/composite-wait-strategy.test.ts index dccb49f8f..c40202970 100644 --- a/src/wait-strategy/composite-wait-strategy.test.ts +++ b/src/wait-strategy/composite-wait-strategy.test.ts @@ -3,7 +3,7 @@ import { AbstractWaitStrategy } from "./wait-strategy"; import { BoundPorts } from "../bound-ports"; import Dockerode from "dockerode"; -const fakeContainer = {} as Dockerode.Container; +const fakeContainer = { id: "containerId" } as Dockerode.Container; const fakeBoundPorts = {} as BoundPorts; const fakeStartTime = {} as Date; @@ -35,21 +35,34 @@ test("should set the startup timeout to the maximum of all wait strategies", asy expect(waitStrategy.getStartupTimeout()).toEqual(2000); }); -test("should override startup timeouts when once is provided", async () => { +test("should override startup timeouts when one is provided", async () => { + const s1 = new FakeWaitStrategy({ resolves: true }).withStartupTimeout(1000); + const s2 = new FakeWaitStrategy({ resolves: true }).withStartupTimeout(2000); + const waitStrategy = new CompositeWaitStrategy([s1, s2]).withStartupTimeout(3000); + + expect(s1.getStartupTimeout()).toEqual(3000); + expect(s2.getStartupTimeout()).toEqual(3000); + expect(waitStrategy.getStartupTimeout()).toEqual(3000); +}); + +test("should enforce startup timeout", async () => { const waitStrategy = new CompositeWaitStrategy([ new FakeWaitStrategy({ resolves: true }).withStartupTimeout(1000), - new FakeWaitStrategy({ resolves: true }).withStartupTimeout(2000), - ]).withStartupTimeout(3000); + new FakeWaitStrategy({ resolves: true }).withStartupTimeout(1000), + ]).withStartupTimeout(0); - expect(waitStrategy.getStartupTimeout()).toEqual(3000); + await expect(waitStrategy.waitUntilReady(fakeContainer, fakeBoundPorts, fakeStartTime)).rejects.toThrowError( + "not resolved after 0ms" + ); }); class FakeWaitStrategy extends AbstractWaitStrategy { - constructor(private opts: { resolves: boolean; error?: Error }) { + constructor(private opts: { resolves: boolean; error?: Error; delay?: number }) { super(); } - waitUntilReady(): Promise { + async waitUntilReady(): Promise { + await new Promise((resolve) => setTimeout(resolve, this.opts.delay ?? 0)); if (this.opts.resolves) { return Promise.resolve(); } else { diff --git a/src/wait-strategy/composite-wait-strategy.ts b/src/wait-strategy/composite-wait-strategy.ts index 21746664c..1df455be4 100644 --- a/src/wait-strategy/composite-wait-strategy.ts +++ b/src/wait-strategy/composite-wait-strategy.ts @@ -1,20 +1,38 @@ import { AbstractWaitStrategy, WaitStrategy } from "./wait-strategy"; import Dockerode from "dockerode"; import { BoundPorts } from "../bound-ports"; +import { log } from "../logger"; export class CompositeWaitStrategy extends AbstractWaitStrategy { constructor(private readonly waitStrategies: WaitStrategy[]) { super(); - this.withStartupTimeout(this.getMaxStartupTimeout()); + this.withStartupTimeout(Math.max(...this.waitStrategies.map((waitStrategy) => waitStrategy.getStartupTimeout()))); } - private getMaxStartupTimeout(): number { - return Math.max(...this.waitStrategies.map((waitStrategy) => waitStrategy.getStartupTimeout())); + public async waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts, startTime?: Date): Promise { + log.debug(`Starting composite wait strategy for ${container.id}`); + + return new Promise((resolve, reject) => { + const onTimeout = () => { + const message = `Composite wait strategy not resolved after ${this.startupTimeout}ms for ${container.id}`; + log.error(message); + reject(new Error(message)); + }; + + const timeout = setTimeout(onTimeout, this.startupTimeout); + + Promise.all( + this.waitStrategies.map((waitStrategy) => waitStrategy.waitUntilReady(container, boundPorts, startTime)) + ) + .then(() => resolve()) + .catch((err) => reject(err)) + .finally(() => clearTimeout(timeout)); + }); } - public async waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts, startTime?: Date): Promise { - await Promise.all( - this.waitStrategies.map((waitStrategy) => waitStrategy.waitUntilReady(container, boundPorts, startTime)) - ); + public override withStartupTimeout(startupTimeout: number): this { + super.withStartupTimeout(startupTimeout); + this.waitStrategies.forEach((waitStrategy) => waitStrategy.withStartupTimeout(startupTimeout)); + return this; } } From d4dc17095557733fa25cb886201559e0af2771d7 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Wed, 12 Apr 2023 10:45:48 +0100 Subject: [PATCH 18/35] Remove unused var from test --- src/wait-strategy/composite-wait-strategy.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/wait-strategy/composite-wait-strategy.test.ts b/src/wait-strategy/composite-wait-strategy.test.ts index c40202970..28c692529 100644 --- a/src/wait-strategy/composite-wait-strategy.test.ts +++ b/src/wait-strategy/composite-wait-strategy.test.ts @@ -57,12 +57,11 @@ test("should enforce startup timeout", async () => { }); class FakeWaitStrategy extends AbstractWaitStrategy { - constructor(private opts: { resolves: boolean; error?: Error; delay?: number }) { + constructor(private opts: { resolves: boolean; error?: Error }) { super(); } - async waitUntilReady(): Promise { - await new Promise((resolve) => setTimeout(resolve, this.opts.delay ?? 0)); + waitUntilReady(): Promise { if (this.opts.resolves) { return Promise.resolve(); } else { From 1bf199ebc14974e1cbc9bfe2d42aef308a4661c0 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Wed, 12 Apr 2023 11:02:50 +0100 Subject: [PATCH 19/35] Re-add delay --- src/wait-strategy/composite-wait-strategy.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/wait-strategy/composite-wait-strategy.test.ts b/src/wait-strategy/composite-wait-strategy.test.ts index 28c692529..c40202970 100644 --- a/src/wait-strategy/composite-wait-strategy.test.ts +++ b/src/wait-strategy/composite-wait-strategy.test.ts @@ -57,11 +57,12 @@ test("should enforce startup timeout", async () => { }); class FakeWaitStrategy extends AbstractWaitStrategy { - constructor(private opts: { resolves: boolean; error?: Error }) { + constructor(private opts: { resolves: boolean; error?: Error; delay?: number }) { super(); } - waitUntilReady(): Promise { + async waitUntilReady(): Promise { + await new Promise((resolve) => setTimeout(resolve, this.opts.delay ?? 0)); if (this.opts.resolves) { return Promise.resolve(); } else { From a0c384c60bf40e964d6bc590aff818473f3c80da Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Wed, 12 Apr 2023 13:09:45 +0100 Subject: [PATCH 20/35] Document composite wait strategy --- docs/features/wait-strategies.md | 52 ++++++++++++++++++- .../composite-wait-strategy.test.ts | 8 +-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/docs/features/wait-strategies.md b/docs/features/wait-strategies.md index 807eee1f5..10566e171 100644 --- a/docs/features/wait-strategies.md +++ b/docs/features/wait-strategies.md @@ -10,7 +10,7 @@ const container = await new GenericContainer("alpine") .start(); ``` -## Host port +## Listening ports The default wait strategy used by Testcontainers. It will wait up to 60 seconds for the container's mapped network ports to be bound. @@ -22,6 +22,17 @@ const container = await new GenericContainer("alpine") .start(); ``` +It can be set explicitly but is not required: + +```javascript +const { GenericContainer, Wait } = require("testcontainers"); + +const container = await new GenericContainer("alpine") + .withExposedPorts(6379) + .withWaitStrategy(Wait.forListeningPorts()) + .start(); +``` + ## Log output Wait until the container has logged a message: @@ -163,6 +174,45 @@ const container = await new GenericContainer("alpine") .start(); ``` +## Composite + +Multiple wait strategies can be chained together: + +```javascript +const { GenericContainer, Wait } = require("testcontainers"); + +const container = await new GenericContainer("alpine") + .withWaitStrategy(Wait.forAll([ + Wait.forListeningPorts(), + Wait.forLogMessage("Ready to accept connections") + ])) + .start(); +``` + +By default, the startup timeout of the composite wait strategy will be the maximum of the startup timeouts of the provided wait strategies. For example: + +```javascript +const w1 = Wait.forListeningPorts().withStartupTimeout(1000); +const w1 = Wait.forLogMessage("").withStartupTimeout(2000); + +const composite = Wait.forAll([w1, w2]); + +expect(composite.getStartupTimeout()).toBe(2000); +``` + +You can override the startup timeouts of the provided wait strategies by setting a startup timeout on the composite. For example: + +```javascript +const w1 = Wait.forListeningPorts().withStartupTimeout(1000); +const w1 = Wait.forLogMessage("").withStartupTimeout(2000); + +const composite = Wait.forAll([w1, w2]).withStartupTimeout(3000) + +expect(w1.getStartupTimeout()).toBe(3000); +expect(w2.getStartupTimeout()).toBe(3000); +expect(composite.getStartupTimeout()).toBe(3000); +``` + ## Other startup strategies If these options do not meet your requirements, you can subclass `StartupCheckStrategy` and use `Dockerode`, which is the underlying Docker client used by Testcontainers: diff --git a/src/wait-strategy/composite-wait-strategy.test.ts b/src/wait-strategy/composite-wait-strategy.test.ts index c40202970..a90e721ea 100644 --- a/src/wait-strategy/composite-wait-strategy.test.ts +++ b/src/wait-strategy/composite-wait-strategy.test.ts @@ -47,8 +47,8 @@ test("should override startup timeouts when one is provided", async () => { test("should enforce startup timeout", async () => { const waitStrategy = new CompositeWaitStrategy([ - new FakeWaitStrategy({ resolves: true }).withStartupTimeout(1000), - new FakeWaitStrategy({ resolves: true }).withStartupTimeout(1000), + new FakeWaitStrategy({ resolves: true, delay: 1000 }), + new FakeWaitStrategy({ resolves: true, delay: 1000 }), ]).withStartupTimeout(0); await expect(waitStrategy.waitUntilReady(fakeContainer, fakeBoundPorts, fakeStartTime)).rejects.toThrowError( @@ -62,7 +62,9 @@ class FakeWaitStrategy extends AbstractWaitStrategy { } async waitUntilReady(): Promise { - await new Promise((resolve) => setTimeout(resolve, this.opts.delay ?? 0)); + if (this.opts.delay) { + await new Promise((resolve) => setTimeout(resolve, this.opts.delay)); + } if (this.opts.resolves) { return Promise.resolve(); } else { From 2ba80d1c66dbe610dcee5d08ad2791ff8c90cafa Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 13 Apr 2023 13:26:46 +0100 Subject: [PATCH 21/35] Use retry mechanism instead of listening ports for Reaper --- .github/workflows/test-main.yml | 28 ++++++++++++++++++++++++++++ src/reaper.ts | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-main.yml b/.github/workflows/test-main.yml index 37dbad12a..aa0bd3479 100644 --- a/.github/workflows/test-main.yml +++ b/.github/workflows/test-main.yml @@ -100,6 +100,34 @@ jobs: CI_ROOTLESS: true CI_PODMAN: true + colima: + runs-on: ${{ matrix.os-version }} + strategy: + fail-fast: false + matrix: + os-version: [ macos-latest ] + node-version: [ 14.x, 16.x, 18.x ] + steps: + - name: Code checkout + uses: actions/checkout@v3 + - name: Start Colima + run: | + brew install docker docker-compose + colima start --cpu 3 --memory 10 --disk 10 --runtime docker # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + colima status + - name: Set environment + run: | + echo "DOCKER_HOST=unix://${HOME}/.colima/default/docker.sock" >> $GITHUB_ENV + echo "TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=/var/run/docker.sock" >> $GITHUB_ENV + - name: Install NodeJS ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: Install dependencies + run: npm ci --omit=optional + - name: Run tests + run: npm run test:ci + smoke-test: runs-on: ${{ matrix.os-version }} strategy: diff --git a/src/reaper.ts b/src/reaper.ts index 0b47ac8f1..9bc357eb2 100644 --- a/src/reaper.ts +++ b/src/reaper.ts @@ -100,7 +100,7 @@ export class ReaperInstance { target: "/var/run/docker.sock", }, ]) - .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage(/.+ Started!/)])); + .withWaitStrategy(Wait.forLogMessage(/.+ Started!/)); if (this.isPrivileged()) { container.withPrivilegedMode(); From 1317d2f305eb4a06c73ae740d319295ca0ecf037 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 13 Apr 2023 14:29:56 +0100 Subject: [PATCH 22/35] Increase reaper retries to 10 --- .github/workflows/test-main.yml | 3 ++- .github/workflows/test.yml | 3 ++- src/reaper.ts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-main.yml b/.github/workflows/test-main.yml index aa0bd3479..8e13a0888 100644 --- a/.github/workflows/test-main.yml +++ b/.github/workflows/test-main.yml @@ -113,7 +113,8 @@ jobs: - name: Start Colima run: | brew install docker docker-compose - colima start --cpu 3 --memory 10 --disk 10 --runtime docker # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + colima start --cpu 3 --memory 10 --disk 10 --runtime docker colima status - name: Set environment run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 740d3fd80..2163a3d64 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,7 +107,8 @@ jobs: - name: Start Colima run: | brew install docker docker-compose - colima start --cpu 3 --memory 10 --disk 10 --runtime docker # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + colima start --cpu 3 --memory 10 --disk 10 --runtime docker colima status - name: Set environment run: | diff --git a/src/reaper.ts b/src/reaper.ts index 9bc357eb2..211b7d39b 100644 --- a/src/reaper.ts +++ b/src/reaper.ts @@ -140,7 +140,7 @@ export class ReaperInstance { }, (result) => result !== undefined, () => new Error(`Failed to connect to Reaper ${containerId}`), - 4000 + 9000 ); if (retryResult instanceof RealReaper) { From ef94d4ef79acc08cfb2c04377990fb937d1bbf98 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 13 Apr 2023 15:23:18 +0100 Subject: [PATCH 23/35] Docker event stream test helper should not fail on invalid JSON --- .../docker-compose-environment.test.ts | 9 +++++++-- .../generic-container-dockerfile.test.ts | 7 +++++-- src/generic-container/generic-container.test.ts | 6 ++++-- src/test-helper.ts | 16 ++++++---------- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/docker-compose-environment/docker-compose-environment.test.ts b/src/docker-compose-environment/docker-compose-environment.test.ts index 97e4d4b72..1c9f20398 100644 --- a/src/docker-compose-environment/docker-compose-environment.test.ts +++ b/src/docker-compose-environment/docker-compose-environment.test.ts @@ -5,6 +5,7 @@ import { Wait } from "../wait-strategy/wait"; import { checkEnvironmentContainerIsHealthy, composeContainerName, + getDockerEventStream, getRunningContainerNames, getVolumeNames, waitForDockerEvent, @@ -44,10 +45,12 @@ describe("DockerComposeEnvironment", () => { const env = new DockerComposeEnvironment(fixtures, "docker-compose-with-many-services.yml"); const startedEnv1 = await env.up(); - const dockerPullEventPromise = waitForDockerEvent("pull", 2); + const dockerEventStream = await getDockerEventStream(); + const dockerPullEventPromise = waitForDockerEvent(dockerEventStream, "pull", 2); const startedEnv2 = await env.withPullPolicy(new AlwaysPullPolicy()).up(); await dockerPullEventPromise; + dockerEventStream.destroy(); await startedEnv1.stop(); await startedEnv2.stop(); }); @@ -56,10 +59,12 @@ describe("DockerComposeEnvironment", () => { const env = new DockerComposeEnvironment(fixtures, "docker-compose-with-many-services.yml"); const startedEnv1 = await env.up(["service_2"]); - const dockerPullEventPromise = waitForDockerEvent("pull"); + const dockerEventStream = await getDockerEventStream(); + const dockerPullEventPromise = waitForDockerEvent(dockerEventStream, "pull"); const startedEnv2 = await env.withPullPolicy(new AlwaysPullPolicy()).up(["service_2"]); await dockerPullEventPromise; + dockerEventStream.destroy(); await startedEnv1.stop(); await startedEnv2.stop(); }); diff --git a/src/generic-container/generic-container-dockerfile.test.ts b/src/generic-container/generic-container-dockerfile.test.ts index 64aba43ec..bb1ea3592 100644 --- a/src/generic-container/generic-container-dockerfile.test.ts +++ b/src/generic-container/generic-container-dockerfile.test.ts @@ -2,7 +2,7 @@ import path from "path"; import { GenericContainer } from "./generic-container"; import { AlwaysPullPolicy } from "../pull-policy"; import { Wait } from "../wait-strategy/wait"; -import { checkContainerIsHealthy, waitForDockerEvent } from "../test-helper"; +import { checkContainerIsHealthy, getDockerEventStream, waitForDockerEvent } from "../test-helper"; describe("GenericContainer Dockerfile", () => { jest.setTimeout(180_000); @@ -26,9 +26,12 @@ describe("GenericContainer Dockerfile", () => { const containerSpec = GenericContainer.fromDockerfile(dockerfile).withPullPolicy(new AlwaysPullPolicy()); await containerSpec.build(); - const dockerPullEventPromise = waitForDockerEvent("pull"); + const dockerEventStream = await getDockerEventStream(); + const dockerPullEventPromise = waitForDockerEvent(dockerEventStream, "pull"); await containerSpec.build(); await dockerPullEventPromise; + + dockerEventStream.destroy(); }); } diff --git a/src/generic-container/generic-container.test.ts b/src/generic-container/generic-container.test.ts index 49462b6a7..9a84e2b62 100644 --- a/src/generic-container/generic-container.test.ts +++ b/src/generic-container/generic-container.test.ts @@ -3,7 +3,7 @@ import path from "path"; import getPort from "get-port"; import { GenericContainer } from "./generic-container"; import { AlwaysPullPolicy } from "../pull-policy"; -import { checkContainerIsHealthy, waitForDockerEvent } from "../test-helper"; +import { checkContainerIsHealthy, getDockerEventStream, waitForDockerEvent } from "../test-helper"; import { getContainerById } from "../docker/functions/container/get-container"; describe("GenericContainer", () => { @@ -223,10 +223,12 @@ describe("GenericContainer", () => { const container = new GenericContainer("cristianrgreco/testcontainer:1.1.14").withExposedPorts(8080); const startedContainer1 = await container.start(); - const dockerPullEventPromise = waitForDockerEvent("pull"); + const dockerEventStream = await getDockerEventStream(); + const dockerPullEventPromise = waitForDockerEvent(dockerEventStream, "pull"); const startedContainer2 = await container.withPullPolicy(new AlwaysPullPolicy()).start(); await dockerPullEventPromise; + dockerEventStream.destroy(); await startedContainer1.stop(); await startedContainer2.stop(); }); diff --git a/src/test-helper.ts b/src/test-helper.ts index 631f1da2c..aea80cc7c 100644 --- a/src/test-helper.ts +++ b/src/test-helper.ts @@ -29,7 +29,7 @@ export const checkEnvironmentContainerIsHealthy = async ( await checkContainerIsHealthy(container); }; -export const getEvents = async (opts: GetEventsOptions = {}): Promise => { +export const getDockerEventStream = async (opts: GetEventsOptions = {}): Promise => { const { dockerode } = await dockerClient(); const events = (await dockerode.getEvents(opts)) as Readable; events.setEncoding("utf-8"); @@ -87,22 +87,18 @@ export const composeContainerName = async (serviceName: string, index = 1): Prom return dockerComposeInfo?.version.startsWith("1.") ? `${serviceName}_${index}` : `${serviceName}-${index}`; }; -export const waitForDockerEvent = async (event: string, times = 1) => { - const events = await getEvents(); - +export const waitForDockerEvent = async (eventStream: Readable, eventName: string, times = 1) => { let currentTimes = 0; - return new Promise((resolve, reject) => { - events.on("data", (data) => { + return new Promise((resolve) => { + eventStream.on("data", (data) => { try { - if (JSON.parse(data).status === event) { + if (JSON.parse(data).status === eventName) { if (++currentTimes === times) { resolve(); - events.destroy(); } } } catch (err) { - reject(`Unexpected err: ${err}`); - events.destroy(); + // ignored } }); }); From c33921fc924c437fee9fc2299b8306acc2e33575 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 13 Apr 2023 16:59:53 +0100 Subject: [PATCH 24/35] Fix flaky test --- ...r-compose-with-healthcheck-with-start-period.yml | 13 +++++++++++++ .../docker-compose-with-healthcheck.yml | 1 - .../docker-compose-environment.test.ts | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 fixtures/docker-compose/docker-compose-with-healthcheck-with-start-period.yml diff --git a/fixtures/docker-compose/docker-compose-with-healthcheck-with-start-period.yml b/fixtures/docker-compose/docker-compose-with-healthcheck-with-start-period.yml new file mode 100644 index 000000000..736e1ad43 --- /dev/null +++ b/fixtures/docker-compose/docker-compose-with-healthcheck-with-start-period.yml @@ -0,0 +1,13 @@ +version: "3.5" + +services: + container: + image: cristianrgreco/testcontainer:1.1.14 + ports: + - 8080 + healthcheck: + test: "curl -f http://localhost:8080/hello-world || exit 1" + interval: 1s + timeout: 3s + retries: 10 + start_period: 10s diff --git a/fixtures/docker-compose/docker-compose-with-healthcheck.yml b/fixtures/docker-compose/docker-compose-with-healthcheck.yml index 7d34f2605..156454608 100644 --- a/fixtures/docker-compose/docker-compose-with-healthcheck.yml +++ b/fixtures/docker-compose/docker-compose-with-healthcheck.yml @@ -10,4 +10,3 @@ services: interval: 1s timeout: 3s retries: 10 - start_period: 1s diff --git a/src/docker-compose-environment/docker-compose-environment.test.ts b/src/docker-compose-environment/docker-compose-environment.test.ts index 1c9f20398..78e2d9d31 100644 --- a/src/docker-compose-environment/docker-compose-environment.test.ts +++ b/src/docker-compose-environment/docker-compose-environment.test.ts @@ -134,7 +134,7 @@ describe("DockerComposeEnvironment", () => { it("should stop the container when the health check wait strategy times out", async () => { await expect( - new DockerComposeEnvironment(fixtures, "docker-compose-with-healthcheck.yml") + new DockerComposeEnvironment(fixtures, "docker-compose-with-healthcheck-with-start-period.yml") .withWaitStrategy(await composeContainerName("container"), Wait.forHealthCheck()) .withStartupTimeout(0) .up() From 7e8bb9ac1c15596cfc186e7646bdbf83822dd8d4 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Fri, 14 Apr 2023 09:23:35 +0100 Subject: [PATCH 25/35] Try to install Docker via GHA --- .github/workflows/test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2163a3d64..c40a4d47c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,12 +104,15 @@ jobs: steps: - name: Code checkout uses: actions/checkout@v3 - - name: Start Colima + - name: Setup Docker + uses: docker-practice/actions-setup-docker@master + - name: Setup Colima run: | - brew install docker docker-compose + # brew install docker docker-compose # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources colima start --cpu 3 --memory 10 --disk 10 --runtime docker colima status + colima --version - name: Set environment run: | echo "DOCKER_HOST=unix://${HOME}/.colima/default/docker.sock" >> $GITHUB_ENV From 804bcf05200eb880be812cb861dbf7a9efa0d976 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Fri, 14 Apr 2023 09:32:18 +0100 Subject: [PATCH 26/35] Revert install Docker action --- .github/workflows/test.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c40a4d47c..1672cc9ad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,12 +104,9 @@ jobs: steps: - name: Code checkout uses: actions/checkout@v3 - - name: Setup Docker - uses: docker-practice/actions-setup-docker@master - name: Setup Colima run: | - # brew install docker docker-compose - # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + brew install docker docker-compose colima start --cpu 3 --memory 10 --disk 10 --runtime docker colima status colima --version From 3ce60fb6e0ee859e029821527afb9185e4222036 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Fri, 14 Apr 2023 10:13:18 +0100 Subject: [PATCH 27/35] Enforce IPv4 --- .github/workflows/test-main.yml | 7 ++++--- .github/workflows/test.yml | 1 + src/reaper.ts | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-main.yml b/.github/workflows/test-main.yml index 8e13a0888..56181b4dc 100644 --- a/.github/workflows/test-main.yml +++ b/.github/workflows/test-main.yml @@ -110,16 +110,17 @@ jobs: steps: - name: Code checkout uses: actions/checkout@v3 - - name: Start Colima + - name: Setup Colima run: | brew install docker docker-compose - # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources - colima start --cpu 3 --memory 10 --disk 10 --runtime docker + colima start --cpu 3 --memory 10 --disk 10 --runtime docker colima status + colima --version - name: Set environment run: | echo "DOCKER_HOST=unix://${HOME}/.colima/default/docker.sock" >> $GITHUB_ENV echo "TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=/var/run/docker.sock" >> $GITHUB_ENV + echo "NODE_OPTIONS=--dns-result-order=ipv4first" >> $GITHUB_ENV - name: Install NodeJS ${{ matrix.node-version }} uses: actions/setup-node@v3 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1672cc9ad..632bd09b8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,6 +114,7 @@ jobs: run: | echo "DOCKER_HOST=unix://${HOME}/.colima/default/docker.sock" >> $GITHUB_ENV echo "TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=/var/run/docker.sock" >> $GITHUB_ENV + echo "NODE_OPTIONS=--dns-result-order=ipv4first" >> $GITHUB_ENV - name: Install NodeJS ${{ matrix.node-version }} uses: actions/setup-node@v3 with: diff --git a/src/reaper.ts b/src/reaper.ts index 211b7d39b..9bc357eb2 100644 --- a/src/reaper.ts +++ b/src/reaper.ts @@ -140,7 +140,7 @@ export class ReaperInstance { }, (result) => result !== undefined, () => new Error(`Failed to connect to Reaper ${containerId}`), - 9000 + 4000 ); if (retryResult instanceof RealReaper) { From edec96e0eb7262e46f18010871d209f72fb9803a Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Fri, 14 Apr 2023 11:59:55 +0100 Subject: [PATCH 28/35] Update pull policy test --- .../docker-compose-environment.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docker-compose-environment/docker-compose-environment.test.ts b/src/docker-compose-environment/docker-compose-environment.test.ts index 78e2d9d31..df1b18e56 100644 --- a/src/docker-compose-environment/docker-compose-environment.test.ts +++ b/src/docker-compose-environment/docker-compose-environment.test.ts @@ -46,7 +46,7 @@ describe("DockerComposeEnvironment", () => { const startedEnv1 = await env.up(); const dockerEventStream = await getDockerEventStream(); - const dockerPullEventPromise = waitForDockerEvent(dockerEventStream, "pull", 2); + const dockerPullEventPromise = waitForDockerEvent(dockerEventStream, "pull"); const startedEnv2 = await env.withPullPolicy(new AlwaysPullPolicy()).up(); await dockerPullEventPromise; From f962918edfb57dc0d641c41531437122a06f435f Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Fri, 14 Apr 2023 14:10:14 +0100 Subject: [PATCH 29/35] Update CompositeWaitStrategy as per GH discussion --- docs/features/wait-strategies.md | 29 +++-- .../docker-compose-environment.ts | 11 +- src/generic-container/generic-container.ts | 17 ++- .../composite-wait-strategy.test.ts | 122 +++++++----------- src/wait-strategy/composite-wait-strategy.ts | 34 +++-- .../health-check-wait-strategy.ts | 4 +- src/wait-strategy/host-port-wait-strategy.ts | 4 +- src/wait-strategy/http-wait-strategy.ts | 4 +- src/wait-strategy/log-wait-strategy.ts | 4 +- src/wait-strategy/shell-wait-strategy.test.ts | 2 +- src/wait-strategy/shell-wait-strategy.ts | 6 +- src/wait-strategy/startup-check-strategy.ts | 4 +- src/wait-strategy/wait-strategy.ts | 8 +- src/wait-strategy/wait.ts | 2 +- 14 files changed, 129 insertions(+), 122 deletions(-) diff --git a/docs/features/wait-strategies.md b/docs/features/wait-strategies.md index 10566e171..52b38f7c1 100644 --- a/docs/features/wait-strategies.md +++ b/docs/features/wait-strategies.md @@ -189,28 +189,37 @@ const container = await new GenericContainer("alpine") .start(); ``` -By default, the startup timeout of the composite wait strategy will be the maximum of the startup timeouts of the provided wait strategies. For example: +The composite wait strategy by default will respect each individual wait strategy's timeouts. For example: ```javascript const w1 = Wait.forListeningPorts().withStartupTimeout(1000); -const w1 = Wait.forLogMessage("").withStartupTimeout(2000); +const w2 = Wait.forLogMessage("READY").withStartupTimeout(2000); -const composite = Wait.forAll([w1, w2]); +const composite = Wait.forAll([w1, w2]) -expect(composite.getStartupTimeout()).toBe(2000); +expect(w1.getStartupTimeout()).toBe(1000); +expect(w2.getStartupTimeout()).toBe(2000); ``` -You can override the startup timeouts of the provided wait strategies by setting a startup timeout on the composite. For example: +The startup timeout of inner wait strategies that have not defined their own startup timeout can be set by setting the startup timeout on the composite: ```javascript const w1 = Wait.forListeningPorts().withStartupTimeout(1000); -const w1 = Wait.forLogMessage("").withStartupTimeout(2000); +const w2 = Wait.forLogMessage("READY"); -const composite = Wait.forAll([w1, w2]).withStartupTimeout(3000) +const composite = Wait.forAll([w1, w2]).withStartupTimeout(2000) -expect(w1.getStartupTimeout()).toBe(3000); -expect(w2.getStartupTimeout()).toBe(3000); -expect(composite.getStartupTimeout()).toBe(3000); +expect(w1.getStartupTimeout()).toBe(1000); +expect(w2.getStartupTimeout()).toBe(2000); +``` + +The startup timeout of all wait strategies can be controlled by setting a deadline on the composite. In this case, the composite will throw unless all inner wait strategies have resolved before the deadline. + +```javascript +const w1 = Wait.forListeningPorts(); +const w2 = Wait.forLogMessage("READY"); + +const composite = Wait.forAll([w1, w2]).withDeadline(2000) ``` ## Other startup strategies diff --git a/src/docker-compose-environment/docker-compose-environment.ts b/src/docker-compose-environment/docker-compose-environment.ts index b1578ad62..31ca99bf6 100644 --- a/src/docker-compose-environment/docker-compose-environment.ts +++ b/src/docker-compose-environment/docker-compose-environment.ts @@ -32,7 +32,7 @@ export class DockerComposeEnvironment { private environment: Environment = {}; private pullPolicy: PullPolicy = new DefaultPullPolicy(); private waitStrategy: { [containerName: string]: WaitStrategy } = {}; - private startupTimeout = 60_000; + private startupTimeout?: number; constructor(composeFilePath: string, composeFiles: string | string[], uuid: Uuid = new RandomUuid()) { this.composeFilePath = composeFilePath; @@ -134,9 +134,12 @@ export class DockerComposeEnvironment { const { host, hostIps } = await dockerClient(); const inspectResult = await inspectContainer(container); const boundPorts = BoundPorts.fromInspectResult(hostIps, inspectResult); - const waitStrategy = ( - this.waitStrategy[containerName] ? this.waitStrategy[containerName] : Wait.forListeningPorts() - ).withStartupTimeout(this.startupTimeout); + const waitStrategy = this.waitStrategy[containerName] + ? this.waitStrategy[containerName] + : Wait.forListeningPorts(); + if (this.startupTimeout !== undefined) { + waitStrategy.withStartupTimeout(this.startupTimeout); + } if (containerLog.enabled()) { (await containerLogs(container)) diff --git a/src/generic-container/generic-container.ts b/src/generic-container/generic-container.ts index 1cc8f9812..d3bd28726 100644 --- a/src/generic-container/generic-container.ts +++ b/src/generic-container/generic-container.ts @@ -48,8 +48,8 @@ export class GenericContainer implements TestContainer { } protected opts: CreateContainerOptions; - protected startupTimeout = 60_000; - protected waitStrategy?: WaitStrategy; + protected startupTimeout?: number; + protected waitStrategy: WaitStrategy = Wait.forListeningPorts(); protected tarToCopy?: archiver.Archiver; protected networkMode?: string; protected networkAliases: string[] = []; @@ -131,13 +131,15 @@ export class GenericContainer implements TestContainer { const { host, hostIps } = await dockerClient(); const inspectResult = await inspectContainer(container); const boundPorts = BoundPorts.fromInspectResult(hostIps, inspectResult).filter(this.opts.exposedPorts); - const waitStrategy = (this.waitStrategy ?? Wait.forListeningPorts()).withStartupTimeout(this.startupTimeout); + if (this.startupTimeout !== undefined) { + this.waitStrategy.withStartupTimeout(this.startupTimeout); + } if (this.containerStarting) { await this.containerStarting(inspectResult, true); } - await waitForContainer(container, waitStrategy, boundPorts); + await waitForContainer(container, this.waitStrategy, boundPorts); const startedContainer = new StartedGenericContainer( container, @@ -145,7 +147,7 @@ export class GenericContainer implements TestContainer { inspectResult, boundPorts, inspectResult.name, - waitStrategy + this.waitStrategy ); if (this.containerStarted) { @@ -197,7 +199,10 @@ export class GenericContainer implements TestContainer { const { host, hostIps } = await dockerClient(); const inspectResult = await inspectContainer(container); const boundPorts = BoundPorts.fromInspectResult(hostIps, inspectResult).filter(this.opts.exposedPorts); - const waitStrategy = (this.waitStrategy ?? Wait.forListeningPorts()).withStartupTimeout(this.startupTimeout); + const waitStrategy = this.waitStrategy ?? Wait.forListeningPorts(); + if (this.startupTimeout !== undefined) { + waitStrategy.withStartupTimeout(this.startupTimeout); + } if (containerLog.enabled()) { (await containerLogs(container)) diff --git a/src/wait-strategy/composite-wait-strategy.test.ts b/src/wait-strategy/composite-wait-strategy.test.ts index a90e721ea..c13e8fc97 100644 --- a/src/wait-strategy/composite-wait-strategy.test.ts +++ b/src/wait-strategy/composite-wait-strategy.test.ts @@ -1,74 +1,50 @@ -import { CompositeWaitStrategy } from "./composite-wait-strategy"; -import { AbstractWaitStrategy } from "./wait-strategy"; -import { BoundPorts } from "../bound-ports"; -import Dockerode from "dockerode"; - -const fakeContainer = { id: "containerId" } as Dockerode.Container; -const fakeBoundPorts = {} as BoundPorts; -const fakeStartTime = {} as Date; - -test("should resolve when all wait strategies resolve", async () => { - const waitStrategy = new CompositeWaitStrategy([ - new FakeWaitStrategy({ resolves: true }), - new FakeWaitStrategy({ resolves: true }), - ]); - - await expect(waitStrategy.waitUntilReady(fakeContainer, fakeBoundPorts, fakeStartTime)).resolves.toBeUndefined(); -}); - -test("should reject when any wait strategy rejects", async () => { - const error = new Error("FAILED"); - const waitStrategy = new CompositeWaitStrategy([ - new FakeWaitStrategy({ resolves: true }), - new FakeWaitStrategy({ resolves: false, error }), - ]); - - await expect(waitStrategy.waitUntilReady(fakeContainer, fakeBoundPorts, fakeStartTime)).rejects.toThrowError(error); -}); - -test("should set the startup timeout to the maximum of all wait strategies", async () => { - const waitStrategy = new CompositeWaitStrategy([ - new FakeWaitStrategy({ resolves: true }).withStartupTimeout(1000), - new FakeWaitStrategy({ resolves: true }).withStartupTimeout(2000), - ]); - - expect(waitStrategy.getStartupTimeout()).toEqual(2000); +import { GenericContainer } from "../generic-container/generic-container"; +import { Wait } from "./wait"; + +describe("CompositeWaitStrategy", () => { + jest.setTimeout(180_000); + + it("should work with individual timeouts", async () => { + const container = new GenericContainer("cristianrgreco/testcontainer:1.1.14").withWaitStrategy( + Wait.forAll([ + Wait.forSuccessfulCommand("exit 1").withStartupTimeout(1000), + Wait.forSuccessfulCommand("exit 2").withStartupTimeout(100), + ]) + ); + + await expect(container.start()).rejects.toThrowError(`Shell command "exit 2" not successful after 100ms`); + }); + + it("should work with outer timeout where inner strategy times out", async () => { + const container = new GenericContainer("cristianrgreco/testcontainer:1.1.14").withWaitStrategy( + Wait.forAll([ + Wait.forSuccessfulCommand("exit 1"), + Wait.forSuccessfulCommand("exit 2").withStartupTimeout(100), + ]).withStartupTimeout(1000) + ); + + await expect(container.start()).rejects.toThrowError(`Shell command "exit 2" not successful after 100ms`); + }); + + it("should work with outer timeout where outer strategy times out", async () => { + const container = new GenericContainer("cristianrgreco/testcontainer:1.1.14").withWaitStrategy( + Wait.forAll([ + Wait.forSuccessfulCommand("exit 1"), + Wait.forSuccessfulCommand("exit 2").withStartupTimeout(1000), + ]).withStartupTimeout(100) + ); + + await expect(container.start()).rejects.toThrowError(`Shell command "exit 1" not successful after 100ms`); + }); + + it("should work with maximum outer timeout", async () => { + const container = new GenericContainer("cristianrgreco/testcontainer:1.1.14").withWaitStrategy( + Wait.forAll([ + Wait.forListeningPorts(), + Wait.forSuccessfulCommand("stat /tmp/never.sock").withStartupTimeout(1000), + ]).withDeadline(100) + ); + + await expect(container.start()).rejects.toThrowError("Composite wait strategy not successful after 100ms"); + }); }); - -test("should override startup timeouts when one is provided", async () => { - const s1 = new FakeWaitStrategy({ resolves: true }).withStartupTimeout(1000); - const s2 = new FakeWaitStrategy({ resolves: true }).withStartupTimeout(2000); - const waitStrategy = new CompositeWaitStrategy([s1, s2]).withStartupTimeout(3000); - - expect(s1.getStartupTimeout()).toEqual(3000); - expect(s2.getStartupTimeout()).toEqual(3000); - expect(waitStrategy.getStartupTimeout()).toEqual(3000); -}); - -test("should enforce startup timeout", async () => { - const waitStrategy = new CompositeWaitStrategy([ - new FakeWaitStrategy({ resolves: true, delay: 1000 }), - new FakeWaitStrategy({ resolves: true, delay: 1000 }), - ]).withStartupTimeout(0); - - await expect(waitStrategy.waitUntilReady(fakeContainer, fakeBoundPorts, fakeStartTime)).rejects.toThrowError( - "not resolved after 0ms" - ); -}); - -class FakeWaitStrategy extends AbstractWaitStrategy { - constructor(private opts: { resolves: boolean; error?: Error; delay?: number }) { - super(); - } - - async waitUntilReady(): Promise { - if (this.opts.delay) { - await new Promise((resolve) => setTimeout(resolve, this.opts.delay)); - } - if (this.opts.resolves) { - return Promise.resolve(); - } else { - return Promise.reject(this.opts.error); - } - } -} diff --git a/src/wait-strategy/composite-wait-strategy.ts b/src/wait-strategy/composite-wait-strategy.ts index 1df455be4..1b3c911b6 100644 --- a/src/wait-strategy/composite-wait-strategy.ts +++ b/src/wait-strategy/composite-wait-strategy.ts @@ -4,35 +4,47 @@ import { BoundPorts } from "../bound-ports"; import { log } from "../logger"; export class CompositeWaitStrategy extends AbstractWaitStrategy { + private deadline?: number; + constructor(private readonly waitStrategies: WaitStrategy[]) { super(); - this.withStartupTimeout(Math.max(...this.waitStrategies.map((waitStrategy) => waitStrategy.getStartupTimeout()))); } public async waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts, startTime?: Date): Promise { log.debug(`Starting composite wait strategy for ${container.id}`); return new Promise((resolve, reject) => { - const onTimeout = () => { - const message = `Composite wait strategy not resolved after ${this.startupTimeout}ms for ${container.id}`; - log.error(message); - reject(new Error(message)); - }; - - const timeout = setTimeout(onTimeout, this.startupTimeout); + let deadlineTimeout: NodeJS.Timeout; + if (this.deadline !== undefined) { + deadlineTimeout = setTimeout(() => { + const message = `Composite wait strategy not successful after ${this.deadline}ms for ${container.id}`; + log.error(message); + reject(new Error(message)); + }, this.deadline); + } Promise.all( this.waitStrategies.map((waitStrategy) => waitStrategy.waitUntilReady(container, boundPorts, startTime)) ) .then(() => resolve()) .catch((err) => reject(err)) - .finally(() => clearTimeout(timeout)); + .finally(() => { + if (deadlineTimeout) { + clearTimeout(deadlineTimeout); + } + }); }); } public override withStartupTimeout(startupTimeout: number): this { - super.withStartupTimeout(startupTimeout); - this.waitStrategies.forEach((waitStrategy) => waitStrategy.withStartupTimeout(startupTimeout)); + this.waitStrategies + .filter((waitStrategy) => waitStrategy.getStartupTimeout() === undefined) + .forEach((waitStrategy) => waitStrategy.withStartupTimeout(startupTimeout)); + return this; + } + + public withDeadline(deadline: number): this { + this.deadline = deadline; return this; } } diff --git a/src/wait-strategy/health-check-wait-strategy.ts b/src/wait-strategy/health-check-wait-strategy.ts index 1395a7132..b649e2ecc 100644 --- a/src/wait-strategy/health-check-wait-strategy.ts +++ b/src/wait-strategy/health-check-wait-strategy.ts @@ -3,7 +3,7 @@ import { log } from "../logger"; import { IntervalRetryStrategy } from "../retry-strategy"; import { HealthCheckStatus } from "../docker/types"; import { inspectContainer } from "../docker/functions/container/inspect-container"; -import { AbstractWaitStrategy } from "./wait-strategy"; +import { AbstractWaitStrategy, DEFAULT_STARTUP_TIMEOUT } from "./wait-strategy"; export class HealthCheckWaitStrategy extends AbstractWaitStrategy { public async waitUntilReady(container: Dockerode.Container): Promise { @@ -16,7 +16,7 @@ export class HealthCheckWaitStrategy extends AbstractWaitStrategy { const timeout = this.startupTimeout; throw new Error(`Health check not healthy after ${timeout}ms for ${container.id}`); }, - this.startupTimeout + this.startupTimeout ?? DEFAULT_STARTUP_TIMEOUT ); if (status !== "healthy") { diff --git a/src/wait-strategy/host-port-wait-strategy.ts b/src/wait-strategy/host-port-wait-strategy.ts index 7f9693273..4270ef5b6 100644 --- a/src/wait-strategy/host-port-wait-strategy.ts +++ b/src/wait-strategy/host-port-wait-strategy.ts @@ -3,7 +3,7 @@ import Dockerode from "dockerode"; import { BoundPorts } from "../bound-ports"; import { log } from "../logger"; import { IntervalRetryStrategy } from "../retry-strategy"; -import { AbstractWaitStrategy } from "./wait-strategy"; +import { AbstractWaitStrategy, DEFAULT_STARTUP_TIMEOUT } from "./wait-strategy"; import { dockerClient } from "../docker/docker-client"; export class HostPortWaitStrategy extends AbstractWaitStrategy { @@ -50,7 +50,7 @@ export class HostPortWaitStrategy extends AbstractWaitStrategy { const timeout = this.startupTimeout; throw new Error(`Port ${port} not bound after ${timeout}ms for ${container.id}`); }, - this.startupTimeout + this.startupTimeout ?? DEFAULT_STARTUP_TIMEOUT ); } } diff --git a/src/wait-strategy/http-wait-strategy.ts b/src/wait-strategy/http-wait-strategy.ts index 5bb44d405..2feccca25 100644 --- a/src/wait-strategy/http-wait-strategy.ts +++ b/src/wait-strategy/http-wait-strategy.ts @@ -1,6 +1,6 @@ import Dockerode from "dockerode"; import { BoundPorts } from "../bound-ports"; -import { AbstractWaitStrategy } from "./wait-strategy"; +import { AbstractWaitStrategy, DEFAULT_STARTUP_TIMEOUT } from "./wait-strategy"; import { IntervalRetryStrategy } from "../retry-strategy"; import fetch, { Response } from "node-fetch"; import https, { Agent } from "https"; @@ -103,7 +103,7 @@ export class HttpWaitStrategy extends AbstractWaitStrategy { () => { throw new Error(`URL ${this.path} not accessible after ${this.startupTimeout}ms for ${container.id}`); }, - this.startupTimeout + this.startupTimeout ?? DEFAULT_STARTUP_TIMEOUT ); } diff --git a/src/wait-strategy/log-wait-strategy.ts b/src/wait-strategy/log-wait-strategy.ts index ed5ef268a..800ade29d 100644 --- a/src/wait-strategy/log-wait-strategy.ts +++ b/src/wait-strategy/log-wait-strategy.ts @@ -3,7 +3,7 @@ import { BoundPorts } from "../bound-ports"; import { log } from "../logger"; import { containerLogs } from "../docker/functions/container/container-logs"; import byline from "byline"; -import { AbstractWaitStrategy } from "./wait-strategy"; +import { AbstractWaitStrategy, DEFAULT_STARTUP_TIMEOUT } from "./wait-strategy"; export type Log = string; @@ -18,7 +18,7 @@ export class LogWaitStrategy extends AbstractWaitStrategy { const stream = await containerLogs(container, { since: startTime }); return new Promise((resolve, reject) => { - const startupTimeout = this.startupTimeout; + const startupTimeout = this.startupTimeout ?? DEFAULT_STARTUP_TIMEOUT; const timeout = setTimeout(() => { const message = `Log message "${this.message}" not received after ${startupTimeout}ms for ${container.id}`; log.error(message); diff --git a/src/wait-strategy/shell-wait-strategy.test.ts b/src/wait-strategy/shell-wait-strategy.test.ts index 0cb48efd8..f202584c4 100644 --- a/src/wait-strategy/shell-wait-strategy.test.ts +++ b/src/wait-strategy/shell-wait-strategy.test.ts @@ -19,6 +19,6 @@ describe("ShellWaitStrategy", () => { .withWaitStrategy(Wait.forSuccessfulCommand("stat /tmp/test.lock")) .withStartupTimeout(1000) .start() - ).rejects.toThrowError("Shell command not successful"); + ).rejects.toThrowError(`Shell command "stat /tmp/test.lock" not successful`); }); }); diff --git a/src/wait-strategy/shell-wait-strategy.ts b/src/wait-strategy/shell-wait-strategy.ts index 2d0899677..48c2d76ad 100644 --- a/src/wait-strategy/shell-wait-strategy.ts +++ b/src/wait-strategy/shell-wait-strategy.ts @@ -1,6 +1,6 @@ import Dockerode from "dockerode"; import { log } from "../logger"; -import { AbstractWaitStrategy } from "./wait-strategy"; +import { AbstractWaitStrategy, DEFAULT_STARTUP_TIMEOUT } from "./wait-strategy"; import { IntervalRetryStrategy } from "../retry-strategy"; import { execContainer } from "../docker/functions/container/exec-container"; import { dockerClient } from "../docker/docker-client"; @@ -29,9 +29,9 @@ export class ShellWaitStrategy extends AbstractWaitStrategy { (exitCode) => exitCode === 0, () => { const timeout = this.startupTimeout; - throw new Error(`Shell command not successful after ${timeout}ms for ${container.id}`); + throw new Error(`Shell command "${this.command}" not successful after ${timeout}ms for ${container.id}`); }, - this.startupTimeout + this.startupTimeout ?? DEFAULT_STARTUP_TIMEOUT ); } } diff --git a/src/wait-strategy/startup-check-strategy.ts b/src/wait-strategy/startup-check-strategy.ts index eb8151983..e477f0c1b 100644 --- a/src/wait-strategy/startup-check-strategy.ts +++ b/src/wait-strategy/startup-check-strategy.ts @@ -1,4 +1,4 @@ -import { AbstractWaitStrategy } from "./wait-strategy"; +import { AbstractWaitStrategy, DEFAULT_STARTUP_TIMEOUT } from "./wait-strategy"; import Dockerode from "dockerode"; import { IntervalRetryStrategy } from "../retry-strategy"; import { dockerClient } from "../docker/docker-client"; @@ -15,7 +15,7 @@ export abstract class StartupCheckStrategy extends AbstractWaitStrategy { async () => await this.checkStartupState(dockerode, container.id), (startupStatus) => startupStatus === "SUCCESS" || startupStatus === "FAIL", () => new Error(`Container not accessible after ${this.startupTimeout}ms for ${container.id}`), - this.startupTimeout + this.startupTimeout ?? DEFAULT_STARTUP_TIMEOUT ); if (startupStatus instanceof Error) { diff --git a/src/wait-strategy/wait-strategy.ts b/src/wait-strategy/wait-strategy.ts index a57103650..331339e98 100644 --- a/src/wait-strategy/wait-strategy.ts +++ b/src/wait-strategy/wait-strategy.ts @@ -1,16 +1,18 @@ import { BoundPorts } from "../bound-ports"; import Dockerode from "dockerode"; +export const DEFAULT_STARTUP_TIMEOUT = 60_000; + export interface WaitStrategy { waitUntilReady(container: Dockerode.Container, boundPorts: BoundPorts, startTime?: Date): Promise; withStartupTimeout(startupTimeout: number): WaitStrategy; - getStartupTimeout(): number; + getStartupTimeout(): number | undefined; } export abstract class AbstractWaitStrategy implements WaitStrategy { - protected startupTimeout = 60_000; + protected startupTimeout?: number; public abstract waitUntilReady( container: Dockerode.Container, @@ -23,7 +25,7 @@ export abstract class AbstractWaitStrategy implements WaitStrategy { return this; } - public getStartupTimeout(): number { + public getStartupTimeout(): number | undefined { return this.startupTimeout; } } diff --git a/src/wait-strategy/wait.ts b/src/wait-strategy/wait.ts index a17df089e..925b040c8 100644 --- a/src/wait-strategy/wait.ts +++ b/src/wait-strategy/wait.ts @@ -7,7 +7,7 @@ import { HostPortWaitStrategy } from "./host-port-wait-strategy"; import { CompositeWaitStrategy } from "./composite-wait-strategy"; export class Wait { - public static forAll(waitStrategies: WaitStrategy[]): WaitStrategy { + public static forAll(waitStrategies: WaitStrategy[]): CompositeWaitStrategy { return new CompositeWaitStrategy(waitStrategies); } From 1362c89dc9050da52af140e6d5155311111276c4 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Fri, 14 Apr 2023 14:13:11 +0100 Subject: [PATCH 30/35] Increase kafka test timeout --- src/modules/kafka/kafka-container.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/kafka/kafka-container.test.ts b/src/modules/kafka/kafka-container.test.ts index ea3264680..1f8dc8004 100644 --- a/src/modules/kafka/kafka-container.test.ts +++ b/src/modules/kafka/kafka-container.test.ts @@ -7,7 +7,7 @@ import * as fs from "fs"; import * as path from "path"; describe("KafkaContainer", () => { - jest.setTimeout(240_000); + jest.setTimeout(300_000); // connectBuiltInZK { it("should connect using in-built zoo-keeper", async () => { From 116aed9cd9cb77ec7a40a2a2f4a2bddec63d1baa Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Fri, 14 Apr 2023 15:56:15 +0100 Subject: [PATCH 31/35] Remove pull stream parser, as it can hide errors --- src/docker/pull-stream-parser.test.ts | 135 ------------------- src/docker/pull-stream-parser.ts | 47 +------ src/wait-strategy/host-port-wait-strategy.ts | 2 +- 3 files changed, 2 insertions(+), 182 deletions(-) delete mode 100644 src/docker/pull-stream-parser.test.ts diff --git a/src/docker/pull-stream-parser.test.ts b/src/docker/pull-stream-parser.test.ts deleted file mode 100644 index 57493ab47..000000000 --- a/src/docker/pull-stream-parser.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { Readable } from "stream"; -import { PullStreamParser } from "./pull-stream-parser"; -import { FakeLogger } from "../logger"; -import { DockerImageName } from "../docker-image-name"; - -describe("PullStreamParser", () => { - let logger: FakeLogger; - let parser: PullStreamParser; - let stream: Readable; - - beforeEach(() => { - logger = new FakeLogger(); - parser = new PullStreamParser(new DockerImageName(undefined, "image", "tag"), logger); - stream = new Readable(); - }); - - it("should wait for stream to end", async () => { - const promise = new Promise((resolve) => parser.consume(stream).then(resolve)); - - stream.push(null); - - await promise; - }); - - it("should log nothing if the message is invalid JSON", async () => { - const promise = new Promise((resolve) => parser.consume(stream).then(resolve)); - - stream.push(`Invalid`); - stream.push(null); - - await promise; - - expect(logger.traceLogs).toHaveLength(0); - expect(logger.warnLogs).toEqual(["Unexpected message format: Invalid"]); - }); - - it("should log status", async () => { - const promise = new Promise((resolve) => parser.consume(stream).then(resolve)); - - stream.push(`{"status":"Pulling from cristianrgreco/testcontainer","id":"id"}\n`); - stream.push(null); - - await promise; - - expect(logger.traceLogs).toEqual(["Pulling image:tag - id - Pulling from cristianrgreco/testcontainer"]); - }); - - it("should not log the same status twice", async () => { - const promise = new Promise((resolve) => parser.consume(stream).then(resolve)); - - stream.push(`{"status":"Pulling from cristianrgreco/testcontainer","id":"id"}\n`); - stream.push(`{"status":"Pulling from cristianrgreco/testcontainer","id":"id"}\n`); - stream.push(null); - - await promise; - - expect(logger.traceLogs).toEqual(["Pulling image:tag - id - Pulling from cristianrgreco/testcontainer"]); - }); - - it("should not log the same status twice per image id", async () => { - const promise = new Promise((resolve) => parser.consume(stream).then(resolve)); - - stream.push(`{"status":"Pulling from cristianrgreco/testcontainer","id":"id1"}\n`); - stream.push(`{"status":"Pulling from cristianrgreco/testcontainer","id":"id2"}\n`); - stream.push(null); - - await promise; - - expect(logger.traceLogs).toEqual([ - "Pulling image:tag - id1 - Pulling from cristianrgreco/testcontainer", - "Pulling image:tag - id2 - Pulling from cristianrgreco/testcontainer", - ]); - }); - - it("should not log image id if not provided", async () => { - const promise = new Promise((resolve) => parser.consume(stream).then(resolve)); - - stream.push(`{"status":"Pulling from cristianrgreco/testcontainer"}\n`); - stream.push(null); - - await promise; - - expect(logger.traceLogs).toEqual(["Pulling image:tag - Pulling from cristianrgreco/testcontainer"]); - }); - - it("should log download progress", async () => { - const promise = new Promise((resolve) => parser.consume(stream).then(resolve)); - - stream.push(`{"status":"Downloading","progressDetail":{"current":5,"total":10},"progress":"...","id":"id"}\n`); - stream.push(null); - - await promise; - - expect(logger.traceLogs).toEqual(["Pulling image:tag - id - Downloading 50%"]); - }); - - it("should round download progress percentage", async () => { - const promise = new Promise((resolve) => parser.consume(stream).then(resolve)); - - stream.push(`{"status":"Downloading","progressDetail":{"current":3,"total":9},"progress":"...","id":"id"}\n`); - stream.push(null); - - await promise; - - expect(logger.traceLogs).toEqual(["Pulling image:tag - id - Downloading 33%"]); - }); - - it("should not log the same download progress twice", async () => { - const promise = new Promise((resolve) => parser.consume(stream).then(resolve)); - - stream.push(`{"status":"Downloading","progressDetail":{"current":5,"total":10},"progress":"...","id":"id"}\n`); - stream.push(`{"status":"Downloading","progressDetail":{"current":5.0001,"total":10},"progress":"...","id":"id"}\n`); - stream.push(null); - - await promise; - - expect(logger.traceLogs).toEqual(["Pulling image:tag - id - Downloading 50%"]); - }); - - it("should log a single line at a time", async () => { - const promise = new Promise((resolve) => parser.consume(stream).then(resolve)); - - stream.push( - `{"status":"Downloading","progressDetail":{"current":5,"total":10},"progress":"...","id":"id"}\r{"status":"Downloading","progressDetail":{"current":8,"total":10},"progress":"...","id":"id"}\n` - ); - stream.push(null); - - await promise; - - expect(logger.traceLogs).toEqual([ - "Pulling image:tag - id - Downloading 50%", - "Pulling image:tag - id - Downloading 80%", - ]); - }); -}); diff --git a/src/docker/pull-stream-parser.ts b/src/docker/pull-stream-parser.ts index a8bc3d351..9d34bd70c 100644 --- a/src/docker/pull-stream-parser.ts +++ b/src/docker/pull-stream-parser.ts @@ -7,54 +7,9 @@ export class PullStreamParser { constructor(private readonly dockerImageName: DockerImageName, private readonly logger: Logger) {} public consume(stream: Readable): Promise { - const messagesById: Map> = new Map(); - const statusesById: Map> = new Map(); - return new Promise((resolve) => { - byline(stream).on("data", (line) => { - try { - const json = JSON.parse(line); - const { id, status } = json; - - const prefix = id - ? `Pulling ${this.dockerImageName.toString()} - ${id}` - : `Pulling ${this.dockerImageName.toString()}`; - - if (status === "Downloading") { - const { current, total } = json.progressDetail; - const percentage = Math.round(100 * (current / total)); - const message = `${prefix} - ${json.status} ${percentage}%`; - const messages = this.getOrElse(messagesById, id, new Set()); - if (!messages.has(message)) { - messages.add(message); - this.logger.trace(message); - } - } else { - const statuses = this.getOrElse(statusesById, id, new Set()); - if (!statuses.has(status)) { - statuses.add(status); - const message = `${prefix} - ${json.status}`; - const messages = this.getOrElse(messagesById, id, new Set()); - if (!messages.has(message)) { - messages.add(message); - this.logger.trace(message); - } - } - } - } catch { - this.logger.warn(`Unexpected message format: ${line}`); - } - }); + byline(stream).on("data", (line) => this.logger.trace(`Pulling ${this.dockerImageName.toString()}: ${line}`)); stream.on("end", resolve); }); } - - private getOrElse(map: Map, key: K, orElse: V): V { - const value = map.get(key); - if (value === undefined) { - map.set(key, orElse); - return orElse; - } - return value; - } } diff --git a/src/wait-strategy/host-port-wait-strategy.ts b/src/wait-strategy/host-port-wait-strategy.ts index 4270ef5b6..5c9028967 100644 --- a/src/wait-strategy/host-port-wait-strategy.ts +++ b/src/wait-strategy/host-port-wait-strategy.ts @@ -47,7 +47,7 @@ export class HostPortWaitStrategy extends AbstractWaitStrategy { () => portCheck.isBound(port), (isBound) => isBound, () => { - const timeout = this.startupTimeout; + const timeout = this.startupTimeout ?? DEFAULT_STARTUP_TIMEOUT; throw new Error(`Port ${port} not bound after ${timeout}ms for ${container.id}`); }, this.startupTimeout ?? DEFAULT_STARTUP_TIMEOUT From 67c0afc1d59126bc13702abe6f3b4f0fb83581f7 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Mon, 17 Apr 2023 14:57:39 +0100 Subject: [PATCH 32/35] Increase colima daemon disk --- .github/workflows/test-main.yml | 3 ++- .github/workflows/test.yml | 2 +- .../docker-compose-environment.test.ts | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-main.yml b/.github/workflows/test-main.yml index 56181b4dc..91d794407 100644 --- a/.github/workflows/test-main.yml +++ b/.github/workflows/test-main.yml @@ -17,6 +17,7 @@ jobs: os-version: [ ubuntu-latest ] node-version: [ 14.x, 16.x, 18.x ] steps: + - name: Code checkout - name: Code checkout uses: actions/checkout@v3 with: @@ -113,7 +114,7 @@ jobs: - name: Setup Colima run: | brew install docker docker-compose - colima start --cpu 3 --memory 10 --disk 10 --runtime docker + colima start --cpu 3 --memory 14 --disk 14 --runtime docker colima status colima --version - name: Set environment diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 632bd09b8..016e94749 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,7 +107,7 @@ jobs: - name: Setup Colima run: | brew install docker docker-compose - colima start --cpu 3 --memory 10 --disk 10 --runtime docker + colima start --cpu 3 --memory 14 --disk 14 --runtime docker colima status colima --version - name: Set environment diff --git a/src/docker-compose-environment/docker-compose-environment.test.ts b/src/docker-compose-environment/docker-compose-environment.test.ts index 78e2d9d31..df1b18e56 100644 --- a/src/docker-compose-environment/docker-compose-environment.test.ts +++ b/src/docker-compose-environment/docker-compose-environment.test.ts @@ -46,7 +46,7 @@ describe("DockerComposeEnvironment", () => { const startedEnv1 = await env.up(); const dockerEventStream = await getDockerEventStream(); - const dockerPullEventPromise = waitForDockerEvent(dockerEventStream, "pull", 2); + const dockerPullEventPromise = waitForDockerEvent(dockerEventStream, "pull"); const startedEnv2 = await env.withPullPolicy(new AlwaysPullPolicy()).up(); await dockerPullEventPromise; From 0e91aaf48895fea417ade0dcc0567b5cc03ec3a3 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Mon, 17 Apr 2023 16:28:13 +0100 Subject: [PATCH 33/35] Increase elasticsearch test timeout --- src/modules/elasticsearch/elasticsearch-container.test.ts | 2 +- src/modules/neo4j/neo4j-container.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/elasticsearch/elasticsearch-container.test.ts b/src/modules/elasticsearch/elasticsearch-container.test.ts index a74b6f1c8..0a2d05c16 100644 --- a/src/modules/elasticsearch/elasticsearch-container.test.ts +++ b/src/modules/elasticsearch/elasticsearch-container.test.ts @@ -2,7 +2,7 @@ import { ElasticsearchContainer } from "./elasticsearch-container"; import { Client } from "@elastic/elasticsearch"; describe("ElasticsearchContainer", () => { - jest.setTimeout(180_000); + jest.setTimeout(240_000); // createIndex { it("should create an index", async () => { diff --git a/src/modules/neo4j/neo4j-container.test.ts b/src/modules/neo4j/neo4j-container.test.ts index b7f9fc092..f007b3be9 100755 --- a/src/modules/neo4j/neo4j-container.test.ts +++ b/src/modules/neo4j/neo4j-container.test.ts @@ -48,7 +48,7 @@ describe("Neo4jContainer", () => { // apoc { it("should have APOC plugin installed", async () => { - const container = await new Neo4jContainer().withApoc().withStartupTimeout(120_000).start(); + const container = await new Neo4jContainer().withApoc().start(); const driver = neo4j.driver( container.getBoltUri(), neo4j.auth.basic(container.getUsername(), container.getPassword()) From 0b248139e83e1df0ad9671fe605840c3e50845b9 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Mon, 17 Apr 2023 17:26:40 +0100 Subject: [PATCH 34/35] Adjust startup timeout/test timeout of all modules --- src/modules/elasticsearch/elasticsearch-container.ts | 2 +- src/modules/kafka/kafka-container.ts | 2 +- src/modules/mongodb/mongodb-container.test.ts | 2 +- src/modules/mysql/mysql-container.test.ts | 2 +- src/modules/neo4j/neo4j-container.test.ts | 2 +- src/modules/neo4j/neo4j-container.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/modules/elasticsearch/elasticsearch-container.ts b/src/modules/elasticsearch/elasticsearch-container.ts index ed6400e9f..d74bc2b26 100644 --- a/src/modules/elasticsearch/elasticsearch-container.ts +++ b/src/modules/elasticsearch/elasticsearch-container.ts @@ -17,7 +17,7 @@ export class ElasticsearchContainer extends GenericContainer { target: "/usr/share/elasticsearch/config/jvm.options.d/elasticsearch-default-memory-vm.options", }, ]) - .withStartupTimeout(120_000); + .withStartupTimeout(180_000); return new StartedElasticsearchContainer(await super.start()); } diff --git a/src/modules/kafka/kafka-container.ts b/src/modules/kafka/kafka-container.ts index d19c46779..e6d53aeea 100644 --- a/src/modules/kafka/kafka-container.ts +++ b/src/modules/kafka/kafka-container.ts @@ -44,7 +44,7 @@ export class KafkaContainer extends GenericContainer { constructor(image = KAFKA_IMAGE) { super(image); - this.withExposedPorts(KAFKA_PORT).withStartupTimeout(180_000).withEnvironment({ + this.withExposedPorts(KAFKA_PORT).withStartupTimeout(240_000).withEnvironment({ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "BROKER:PLAINTEXT,PLAINTEXT:PLAINTEXT", KAFKA_INTER_BROKER_LISTENER_NAME: "BROKER", KAFKA_BROKER_ID: "1", diff --git a/src/modules/mongodb/mongodb-container.test.ts b/src/modules/mongodb/mongodb-container.test.ts index 8025be079..4056564b6 100644 --- a/src/modules/mongodb/mongodb-container.test.ts +++ b/src/modules/mongodb/mongodb-container.test.ts @@ -2,7 +2,7 @@ import { MongoDBContainer, StartedMongoDBContainer } from "./mongodb-container"; import mongoose from "mongoose"; describe("MongodbContainer", () => { - jest.setTimeout(240_000); + jest.setTimeout(180_000); // connect4 { it("should work using default version 4.0.1", async () => { diff --git a/src/modules/mysql/mysql-container.test.ts b/src/modules/mysql/mysql-container.test.ts index 214db2a6f..80437bfc4 100644 --- a/src/modules/mysql/mysql-container.test.ts +++ b/src/modules/mysql/mysql-container.test.ts @@ -2,7 +2,7 @@ import { createConnection } from "mysql2/promise"; import { MySqlContainer } from "./mysql-container"; describe("MySqlContainer", () => { - jest.setTimeout(240_000); + jest.setTimeout(180_000); // connect { it("should connect and execute query", async () => { diff --git a/src/modules/neo4j/neo4j-container.test.ts b/src/modules/neo4j/neo4j-container.test.ts index f007b3be9..99269cff3 100755 --- a/src/modules/neo4j/neo4j-container.test.ts +++ b/src/modules/neo4j/neo4j-container.test.ts @@ -2,7 +2,7 @@ import neo4j from "neo4j-driver"; import { Neo4jContainer } from "./neo4j-container"; describe("Neo4jContainer", () => { - jest.setTimeout(180_000); + jest.setTimeout(240_000); // createNode { it("should create a person node", async () => { diff --git a/src/modules/neo4j/neo4j-container.ts b/src/modules/neo4j/neo4j-container.ts index f09444b1c..b279214de 100755 --- a/src/modules/neo4j/neo4j-container.ts +++ b/src/modules/neo4j/neo4j-container.ts @@ -34,7 +34,7 @@ export class Neo4jContainer extends GenericContainer { this.withExposedPorts(...(this.hasExposedPorts ? this.opts.exposedPorts : [BOLT_PORT, HTTP_PORT])) .withWaitStrategy(Wait.forAll([Wait.forListeningPorts(), Wait.forLogMessage("Started.")])) .withEnvironment({ NEO4J_AUTH: `${USERNAME}/${this.password}` }) - .withStartupTimeout(120_000); + .withStartupTimeout(180_000); if (this.apoc) { this.withEnvironment({ From 7074f7ee0a0e5057009e0b28481f7b09286e42b3 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Tue, 18 Apr 2023 08:55:52 +0100 Subject: [PATCH 35/35] Increase startup timeouts of Kafka/MySQL modules --- src/modules/kafka/kafka-container.test.ts | 2 +- src/modules/kafka/kafka-container.ts | 2 +- src/modules/mysql/mysql-container.test.ts | 2 +- src/modules/mysql/mysql-container.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/modules/kafka/kafka-container.test.ts b/src/modules/kafka/kafka-container.test.ts index 1f8dc8004..20886a5a9 100644 --- a/src/modules/kafka/kafka-container.test.ts +++ b/src/modules/kafka/kafka-container.test.ts @@ -7,7 +7,7 @@ import * as fs from "fs"; import * as path from "path"; describe("KafkaContainer", () => { - jest.setTimeout(300_000); + jest.setTimeout(360_000); // connectBuiltInZK { it("should connect using in-built zoo-keeper", async () => { diff --git a/src/modules/kafka/kafka-container.ts b/src/modules/kafka/kafka-container.ts index e6d53aeea..8c49ad052 100644 --- a/src/modules/kafka/kafka-container.ts +++ b/src/modules/kafka/kafka-container.ts @@ -44,7 +44,7 @@ export class KafkaContainer extends GenericContainer { constructor(image = KAFKA_IMAGE) { super(image); - this.withExposedPorts(KAFKA_PORT).withStartupTimeout(240_000).withEnvironment({ + this.withExposedPorts(KAFKA_PORT).withStartupTimeout(300_000).withEnvironment({ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "BROKER:PLAINTEXT,PLAINTEXT:PLAINTEXT", KAFKA_INTER_BROKER_LISTENER_NAME: "BROKER", KAFKA_BROKER_ID: "1", diff --git a/src/modules/mysql/mysql-container.test.ts b/src/modules/mysql/mysql-container.test.ts index 80437bfc4..214db2a6f 100644 --- a/src/modules/mysql/mysql-container.test.ts +++ b/src/modules/mysql/mysql-container.test.ts @@ -2,7 +2,7 @@ import { createConnection } from "mysql2/promise"; import { MySqlContainer } from "./mysql-container"; describe("MySqlContainer", () => { - jest.setTimeout(180_000); + jest.setTimeout(240_000); // connect { it("should connect and execute query", async () => { diff --git a/src/modules/mysql/mysql-container.ts b/src/modules/mysql/mysql-container.ts index 834c74fb7..085b8de24 100644 --- a/src/modules/mysql/mysql-container.ts +++ b/src/modules/mysql/mysql-container.ts @@ -42,7 +42,7 @@ export class MySqlContainer extends GenericContainer { MYSQL_USER: this.username, MYSQL_PASSWORD: this.userPassword, }) - .withStartupTimeout(120_000); + .withStartupTimeout(180_000); return new StartedMySqlContainer( await super.start(),