feat: Add reuse support (#1051)

Co-authored-by: Andre Hofmeister <9199345+HofmeisterAn@users.noreply.github.com>

Author: david-szabo97

docs/api/resource-reaper.md docs/api/resource_reaper.md

@@ -0,0 +1,22 @@
+# Resource Reuse
+
+Reuse is an experimental feature designed to simplify and enhance the development experience. Instead of disposing resources after the tests are finished, enabling reuse will retain the resources and reuse them in the next test run. Testcontainers assigns a hash value according to the builder configuration. If it identifies a matching resource, it will reuse this resource instead of creating a new one. Enabling reuse will disable the resource reaper, meaning the resource will not be cleaned up.
+
+```csharp title="Enable container reuse"
+_ = new ContainerBuilder()
+  .WithReuse(true);
+```
+
+The reuse implementation does not currently consider (support) all builder APIs when calculating the hash value. Therefore, collisions may occur. To prevent collisions, simply use a distinct label to identify the resource.
+
+```csharp title="Label container resource to identify it"
+_ = new ContainerBuilder()
+  .WithReuse(true)
+  .WithLabel("reuse-id", "WeatherForecast");
+```
+
+!!!warning
+
+    Reuse does not replace singleton implementations to improve test performance. Prefer proper shared instances according to your chosen test framework.
+
+Calling `Dispose()` on a reusable container will stop it. Testcontainers will automatically start it in the next test run. This will assign a new random host port. Some services (e.g. Kafka) require the random assigned host port on the initial configuration. This may interfere with the new random assigned host port.

docs/api/resource_reuse.md

@@ -31,7 +31,8 @@ nav:
   - api/create_docker_image.md
   - api/create_docker_container.md
   - api/create_docker_network.md
-  - api/resource-reaper.md
+  - api/resource_reaper.md
+  # - api/resource_reuse.md
   - api/wait_strategies.md
   - api/best_practices.md
   - Examples:

mkdocs.yml

@@ -57,6 +57,12 @@ public TBuilderEntity WithCleanUp(bool cleanUp)
       return WithResourceReaperSessionId(TestcontainersSettings.ResourceReaperEnabled && cleanUp ? ResourceReaper.DefaultSessionId : Guid.Empty);
     }
 
+    /// <inheritdoc />
+    public TBuilderEntity WithReuse(bool reuse)
+    {
+      return Clone(new ResourceConfiguration<TCreateResourceEntity>(reuse: reuse)).WithCleanUp(!reuse);
+    }
+
     /// <inheritdoc />
     public TBuilderEntity WithLabel(string name, string value)
     {
@@ -126,9 +132,13 @@ protected virtual TBuilderEntity Init()
     /// <exception cref="ArgumentException">Thrown when a mandatory Docker resource configuration is not set.</exception>
     protected virtual void Validate()
     {
-      const string message = "Docker is either not running or misconfigured. Please ensure that Docker is running and that the endpoint is properly configured. You can customize your configuration using either the environment variables or the ~/.testcontainers.properties file. For more information, visit:\nhttps://dotnet.testcontainers.org/custom_configuration/";
+      const string containerRuntimeNotFound = "Docker is either not running or misconfigured. Please ensure that Docker is running and that the endpoint is properly configured. You can customize your configuration using either the environment variables or the ~/.testcontainers.properties file. For more information, visit:\nhttps://dotnet.testcontainers.org/custom_configuration/";
       _ = Guard.Argument(DockerResourceConfiguration.DockerEndpointAuthConfig, nameof(IResourceConfiguration<TCreateResourceEntity>.DockerEndpointAuthConfig))
-        .ThrowIf(argument => argument.Value == null, argument => new ArgumentException(message, argument.Name));
+        .ThrowIf(argument => argument.Value == null, argument => new ArgumentException(containerRuntimeNotFound, argument.Name));
+
+      const string reuseNotSupported = "Reuse cannot be used in conjunction with WithCleanUp(true).";
+      _ = Guard.Argument(DockerResourceConfiguration, nameof(IResourceConfiguration<TCreateResourceEntity>.Reuse))
+        .ThrowIf(argument => argument.Value.Reuse.HasValue && argument.Value.Reuse.Value && !Guid.Empty.Equals(argument.Value.SessionId), argument => new ArgumentException(reuseNotSupported, argument.Name));
     }
 
     /// <summary>

src/Testcontainers/Builders/AbstractBuilder`4.cs

@@ -364,6 +364,10 @@ protected override void Validate()
     {
       base.Validate();
 
+      const string reuseNotSupported = "Reuse cannot be used in conjunction with WithAutoRemove(true).";
+      _ = Guard.Argument(DockerResourceConfiguration, nameof(IContainerConfiguration.Reuse))
+        .ThrowIf(argument => argument.Value.Reuse.HasValue && argument.Value.Reuse.Value && argument.Value.AutoRemove.HasValue && argument.Value.AutoRemove.Value, argument => new ArgumentException(reuseNotSupported, argument.Name));
+
       _ = Guard.Argument(DockerResourceConfiguration.Image, nameof(IContainerConfiguration.Image))
         .NotNull();
     }

src/Testcontainers/Builders/ContainerBuilder`3.cs

@@ -59,6 +59,26 @@ public interface IAbstractBuilder<out TBuilderEntity, out TResourceEntity, out T
     [PublicAPI]
     TBuilderEntity WithCleanUp(bool cleanUp);
 
+    /// <summary>
+    /// Reuses an existing Docker resource.
+    /// </summary>
+    /// <remarks>
+    /// If reuse is enabled, Testcontainers will label the resource with a hash value
+    /// according to the respective build/resource configuration. When Testcontainers finds a
+    /// matching resource, it will reuse this resource instead of creating a new one. Enabling
+    /// reuse will disable the resource reaper, meaning the resource will not be cleaned up
+    /// after the tests are finished.
+    ///
+    /// This is an <b>experimental</b> feature. Reuse does not take all builder
+    /// configurations into account when calculating the hash value. There might be configurations
+    /// where Testcontainers is not, or not yet, able to find a matching resource and
+    /// recreate the resource.
+    /// </remarks>
+    /// <param name="reuse">Determines whether to reuse an existing resource configuration or not.</param>
+    /// <returns>A configured instance of <typeparamref name="TBuilderEntity" />.</returns>
+    [PublicAPI]
+    TBuilderEntity WithReuse(bool reuse);
+
     /// <summary>
     /// Adds user-defined metadata to the Docker resource.
     /// </summary>

src/Testcontainers/Builders/IAbstractBuilder`3.cs

@@ -112,6 +112,16 @@ protected sealed override ImageFromDockerfileBuilder Init()
       return base.Init().WithImageBuildPolicy(PullPolicy.Always).WithDockerfile("Dockerfile").WithDockerfileDirectory(Directory.GetCurrentDirectory()).WithName(new DockerImage("localhost/testcontainers", Guid.NewGuid().ToString("D"), string.Empty));
     }
 
+    /// <inheritdoc />
+    protected override void Validate()
+    {
+      base.Validate();
+
+      const string reuseNotSupported = "Building an image does not support the reuse feature. To keep the built image, disable the cleanup.";
+      _ = Guard.Argument(DockerResourceConfiguration, nameof(IImageFromDockerfileConfiguration.Reuse))
+        .ThrowIf(argument => argument.Value.Reuse.HasValue && argument.Value.Reuse.Value, argument => new ArgumentException(reuseNotSupported, argument.Name));
+    }
+
     /// <inheritdoc />
     protected override ImageFromDockerfileBuilder Clone(IResourceConfiguration<ImageBuildParameters> resourceConfiguration)
     {

src/Testcontainers/Builders/ImageFromDockerfileBuilder.cs

@@ -200,6 +200,11 @@ public async Task<string> RunAsync(IContainerConfiguration configuration, Cancel
         NetworkingConfig = networkingConfig,
       };
 
+      if (configuration.Reuse.HasValue && configuration.Reuse.Value)
+      {
+        createParameters.Labels.Add(TestcontainersClient.TestcontainersReuseHashLabel, configuration.GetReuseHash());
+      }
+
       if (configuration.ParameterModifiers != null)
       {
         foreach (var parameterModifier in configuration.ParameterModifiers)

src/Testcontainers/Clients/DockerContainerOperations.cs

@@ -63,6 +63,11 @@ public async Task<string> CreateAsync(INetworkConfiguration configuration, Cance
         Labels = configuration.Labels.ToDictionary(item => item.Key, item => item.Value),
       };
 
+      if (configuration.Reuse.HasValue && configuration.Reuse.Value)
+      {
+        createParameters.Labels.Add(TestcontainersClient.TestcontainersReuseHashLabel, configuration.GetReuseHash());
+      }
+
       if (configuration.ParameterModifiers != null)
       {
         foreach (var parameterModifier in configuration.ParameterModifiers)

src/Testcontainers/Clients/DockerNetworkOperations.cs

@@ -65,6 +65,11 @@ public async Task<string> CreateAsync(IVolumeConfiguration configuration, Cancel
         Labels = configuration.Labels.ToDictionary(item => item.Key, item => item.Value),
       };
 
+      if (configuration.Reuse.HasValue && configuration.Reuse.Value)
+      {
+        createParameters.Labels.Add(TestcontainersClient.TestcontainersReuseHashLabel, configuration.GetReuseHash());
+      }
+
       if (configuration.ParameterModifiers != null)
       {
         foreach (var parameterModifier in configuration.ParameterModifiers)

src/Testcontainers/Clients/DockerVolumeOperations.cs

@@ -2,8 +2,9 @@ namespace DotNet.Testcontainers.Clients
 {
   using System.Collections.Concurrent;
   using System.Collections.Generic;
+  using DotNet.Testcontainers.Configurations;
 
-  internal sealed class FilterByProperty : ConcurrentDictionary<string, IDictionary<string, bool>>
+  public class FilterByProperty : ConcurrentDictionary<string, IDictionary<string, bool>>
   {
     public FilterByProperty Add(string property, string value)
     {
@@ -12,4 +13,27 @@ public FilterByProperty Add(string property, string value)
       return this;
     }
   }
+
+  public sealed class FilterByReuseHash : FilterByProperty
+  {
+    public FilterByReuseHash(IContainerConfiguration resourceConfiguration)
+      : this(resourceConfiguration.GetReuseHash())
+    {
+    }
+
+    public FilterByReuseHash(INetworkConfiguration resourceConfiguration)
+      : this(resourceConfiguration.GetReuseHash())
+    {
+    }
+
+    public FilterByReuseHash(IVolumeConfiguration resourceConfiguration)
+      : this(resourceConfiguration.GetReuseHash())
+    {
+    }
+
+    private FilterByReuseHash(string hash)
+    {
+      Add("label", string.Join("=", TestcontainersClient.TestcontainersReuseHashLabel, hash));
+    }
+  }
 }

src/Testcontainers/Clients/FilterByProperty.cs

@@ -27,6 +27,8 @@ internal sealed class TestcontainersClient : ITestcontainersClient
 
     public const string TestcontainersSessionIdLabel = TestcontainersLabel + ".session-id";
 
+    public const string TestcontainersReuseHashLabel = TestcontainersLabel + ".reuse-hash";
+
     public static readonly string Version = typeof(TestcontainersClient).Assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>().InformationalVersion;
 
     private static readonly string OSRootDirectory = Path.GetPathRoot(Directory.GetCurrentDirectory());

src/Testcontainers/Clients/TestcontainersClient.cs

@@ -2,7 +2,6 @@ namespace DotNet.Testcontainers.Configurations
 {
   using System;
   using System.Collections.Generic;
-  using System.Collections.ObjectModel;
   using Docker.DotNet;
   using DotNet.Testcontainers.Clients;
   using JetBrains.Annotations;

src/Testcontainers/Configurations/AuthConfigs/DockerEndpointAuthenticationConfiguration.cs

@@ -16,6 +16,11 @@ public interface IResourceConfiguration<in TCreateResourceEntity>
     /// </summary>
     Guid SessionId { get; }
 
+    /// <summary>
+    /// Gets a value indicating whether to reuse an existing resource configuration or not.
+    /// </summary>
+    bool? Reuse { get; }
+
     /// <summary>
     /// Gets the Docker endpoint authentication configuration.
     /// </summary>
@@ -30,5 +35,10 @@ public interface IResourceConfiguration<in TCreateResourceEntity>
     /// Gets a list of low level modifications of the Docker.DotNet entity.
     /// </summary>
     IReadOnlyList<Action<TCreateResourceEntity>> ParameterModifiers { get; }
+
+    /// <summary>
+    /// Gets the reuse hash.
+    /// </summary>
+    string GetReuseHash();
   }
 }

src/Testcontainers/Configurations/Commons/IResourceConfiguration.cs

@@ -0,0 +1,39 @@
+namespace DotNet.Testcontainers.Configurations
+{
+  using System;
+  using System.Collections.Generic;
+  using System.Linq;
+  using System.Text.Json;
+  using System.Text.Json.Serialization;
+  using DotNet.Testcontainers.Clients;
+  using DotNet.Testcontainers.Containers;
+
+  internal sealed class JsonIgnoreRuntimeResourceLabels : JsonConverter<IReadOnlyDictionary<string, string>>
+  {
+    private static readonly ISet<string> IgnoreLabels = new HashSet<string> { ResourceReaper.ResourceReaperSessionLabel, TestcontainersClient.TestcontainersVersionLabel, TestcontainersClient.TestcontainersSessionIdLabel };
+
+    public override bool CanConvert(Type typeToConvert)
+    {
+      return typeof(IEnumerable<KeyValuePair<string, string>>).IsAssignableFrom(typeToConvert);
+    }
+
+    public override IReadOnlyDictionary<string, string> Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+      return JsonSerializer.Deserialize<IReadOnlyDictionary<string, string>>(ref reader);
+    }
+
+    public override void Write(Utf8JsonWriter writer, IReadOnlyDictionary<string, string> value, JsonSerializerOptions options)
+    {
+      var labels = value.Where(label => !IgnoreLabels.Contains(label.Key)).ToDictionary(label => label.Key, label => label.Value);
+
+      writer.WriteStartObject();
+
+      foreach (var label in labels)
+      {
+        writer.WriteString(label.Key, label.Value);
+      }
+
+      writer.WriteEndObject();
+    }
+  }
+}

src/Testcontainers/Configurations/Commons/JsonIgnoreRuntimeResourceLabels.cs

@@ -2,6 +2,9 @@ namespace DotNet.Testcontainers.Configurations
 {
   using System;
   using System.Collections.Generic;
+  using System.Security.Cryptography;
+  using System.Text.Json;
+  using System.Text.Json.Serialization;
   using DotNet.Testcontainers.Builders;
   using DotNet.Testcontainers.Containers;
   using JetBrains.Annotations;
@@ -16,15 +19,18 @@ public class ResourceConfiguration<TCreateResourceEntity> : IResourceConfigurati
     /// <param name="dockerEndpointAuthenticationConfiguration">The Docker endpoint authentication configuration.</param>
     /// <param name="labels">The test session id.</param>
     /// <param name="parameterModifiers">A list of low level modifications of the Docker.DotNet entity.</param>
+    /// <param name="reuse">A value indicating whether to reuse an existing resource configuration or not.</param>
     public ResourceConfiguration(
       IDockerEndpointAuthenticationConfiguration dockerEndpointAuthenticationConfiguration = null,
       IReadOnlyDictionary<string, string> labels = null,
-      IReadOnlyList<Action<TCreateResourceEntity>> parameterModifiers = null)
+      IReadOnlyList<Action<TCreateResourceEntity>> parameterModifiers = null,
+      bool? reuse = null)
     {
       SessionId = labels != null && labels.TryGetValue(ResourceReaper.ResourceReaperSessionLabel, out var resourceReaperSessionId) && Guid.TryParseExact(resourceReaperSessionId, "D", out var sessionId) ? sessionId : Guid.Empty;
       DockerEndpointAuthConfig = dockerEndpointAuthenticationConfiguration;
       Labels = labels;
       ParameterModifiers = parameterModifiers;
+      Reuse = reuse;
     }
 
     /// <summary>
@@ -44,21 +50,41 @@ protected ResourceConfiguration(IResourceConfiguration<TCreateResourceEntity> re
     protected ResourceConfiguration(IResourceConfiguration<TCreateResourceEntity> oldValue, IResourceConfiguration<TCreateResourceEntity> newValue)
       : this(
         dockerEndpointAuthenticationConfiguration: BuildConfiguration.Combine(oldValue.DockerEndpointAuthConfig, newValue.DockerEndpointAuthConfig),
+        labels: BuildConfiguration.Combine(oldValue.Labels, newValue.Labels),
         parameterModifiers: BuildConfiguration.Combine(oldValue.ParameterModifiers, newValue.ParameterModifiers),
-        labels: BuildConfiguration.Combine(oldValue.Labels, newValue.Labels))
+        reuse: (oldValue.Reuse.HasValue && oldValue.Reuse.Value) || (newValue.Reuse.HasValue && newValue.Reuse.Value))
     {
     }
 
     /// <inheritdoc />
+    [JsonIgnore]
     public Guid SessionId { get; }
 
     /// <inheritdoc />
+    [JsonIgnore]
+    public bool? Reuse { get; }
+
+    /// <inheritdoc />
+    [JsonIgnore]
     public IDockerEndpointAuthenticationConfiguration DockerEndpointAuthConfig { get; }
 
     /// <inheritdoc />
+    [JsonConverter(typeof(JsonIgnoreRuntimeResourceLabels))]
     public IReadOnlyDictionary<string, string> Labels { get; }
 
     /// <inheritdoc />
+    [JsonIgnore]
     public IReadOnlyList<Action<TCreateResourceEntity>> ParameterModifiers { get; }
+
+    /// <inheritdoc />
+    public virtual string GetReuseHash()
+    {
+      var jsonUtf8Bytes = JsonSerializer.SerializeToUtf8Bytes(this, GetType());
+
+      using (var sha1 = SHA1.Create())
+      {
+        return Convert.ToBase64String(sha1.ComputeHash(jsonUtf8Bytes));
+      }
+    }
   }
 }