fix: Do not pre pull Dockerfile build stages that do not correspond to base images (#979)

Author: HofmeisterAn

.devcontainer/devcontainer.json

@@ -14,18 +14,20 @@
       "moby": true
     },
     "ghcr.io/devcontainers/features/dotnet:1": {
-      "version": "6.0.405",
+      "version": "6.0.413",
       "installUsingApt": false
     }
   },
-  "extensions": [
-    "formulahendry.dotnet-test-explorer",
-    "ms-azuretools.vscode-docker",
-    "ms-dotnettools.csharp"
-  ],
-  "settings": {
-    "omnisharp.path": "latest" // https://github.com/OmniSharp/omnisharp-vscode/issues/5410#issuecomment-1284531542.
+  "customizations": {
+    "extensions": [
+      "formulahendry.dotnet-test-explorer",
+      "ms-azuretools.vscode-docker",
+      "ms-dotnettools.csharp"
+    ],
+    "settings": {
+      "omnisharp.path": "latest" // https://github.com/OmniSharp/omnisharp-vscode/issues/5410#issuecomment-1284531542.
+    }
   },
-  "postCreateCommand": ["git", "lfs", "pull"],
+  "postCreateCommand": "git config --global --add safe.directory ${containerWorkspaceFolder} && git lfs checkout",
   "postStartCommand": ["dotnet", "build"]
 }

src/Testcontainers/Clients/DockerImageOperations.cs

@@ -88,12 +88,10 @@ public Task DeleteAsync(IImage image, CancellationToken ct = default)
       return Docker.Images.DeleteImageAsync(image.FullName, new ImageDeleteParameters { Force = true }, ct);
     }
 
-    public async Task<string> BuildAsync(IImageFromDockerfileConfiguration configuration, CancellationToken ct = default)
+    public async Task<string> BuildAsync(IImageFromDockerfileConfiguration configuration, ITarArchive dockerfileArchive, CancellationToken ct = default)
     {
       var image = configuration.Image;
 
-      ITarArchive dockerfileArchive = new DockerfileArchive(configuration.DockerfileDirectory, configuration.Dockerfile, image, _logger);
-
       var imageExists = await ExistsWithNameAsync(image.FullName, ct)
         .ConfigureAwait(false);
 

src/Testcontainers/Clients/IDockerImageOperations.cs

@@ -12,6 +12,6 @@ internal interface IDockerImageOperations : IHasListOperations<ImagesListRespons
 
     Task DeleteAsync(IImage image, CancellationToken ct = default);
 
-    Task<string> BuildAsync(IImageFromDockerfileConfiguration configuration, CancellationToken ct = default);
+    Task<string> BuildAsync(IImageFromDockerfileConfiguration configuration, ITarArchive dockerfileArchive, CancellationToken ct = default);
   }
 }

src/Testcontainers/Clients/TestcontainersClient.cs

@@ -6,7 +6,6 @@ namespace DotNet.Testcontainers.Clients
   using System.Linq;
   using System.Reflection;
   using System.Text;
-  using System.Text.RegularExpressions;
   using System.Threading;
   using System.Threading.Tasks;
   using Docker.DotNet;
@@ -33,10 +32,10 @@ internal sealed class TestcontainersClient : ITestcontainersClient
 
     private static readonly string OSRootDirectory = Path.GetPathRoot(Directory.GetCurrentDirectory());
 
-    private static readonly Regex FromLinePattern = new Regex("FROM (?<arg>--[^\\s]+\\s)*(?<image>[^\\s]+).*", RegexOptions.None, TimeSpan.FromSeconds(1));
-
     private readonly DockerRegistryAuthenticationProvider _registryAuthenticationProvider;
 
+    private readonly ILogger _logger;
+
     /// <summary>
     /// Initializes a new instance of the <see cref="TestcontainersClient" /> class.
     /// </summary>
@@ -50,7 +49,8 @@ public TestcontainersClient(Guid sessionId, IDockerEndpointAuthenticationConfigu
         new DockerNetworkOperations(sessionId, dockerEndpointAuthConfig, logger),
         new DockerVolumeOperations(sessionId, dockerEndpointAuthConfig, logger),
         new DockerSystemOperations(sessionId, dockerEndpointAuthConfig, logger),
-        new DockerRegistryAuthenticationProvider(logger))
+        new DockerRegistryAuthenticationProvider(logger),
+        logger)
     {
     }
 
@@ -60,9 +60,11 @@ private TestcontainersClient(
       IDockerNetworkOperations networkOperations,
       IDockerVolumeOperations volumeOperations,
       IDockerSystemOperations systemOperations,
-      DockerRegistryAuthenticationProvider registryAuthenticationProvider)
+      DockerRegistryAuthenticationProvider registryAuthenticationProvider,
+      ILogger logger)
     {
       _registryAuthenticationProvider = registryAuthenticationProvider;
+      _logger = logger;
       Container = containerOperations;
       Image = imageOperations;
       Network = networkOperations;
@@ -328,25 +330,17 @@ await Task.WhenAll(configuration.ResourceMappings.Values.Select(resourceMapping
     /// <inheritdoc />
     public async Task<string> BuildAsync(IImageFromDockerfileConfiguration configuration, CancellationToken ct = default)
     {
-      var dockerfileFilePath = Path.Combine(configuration.DockerfileDirectory, configuration.Dockerfile);
-
       var cachedImage = await Image.ByNameAsync(configuration.Image.FullName, ct)
         .ConfigureAwait(false);
 
-      if (File.Exists(dockerfileFilePath))
-      {
-        await Task.WhenAll(File.ReadAllLines(dockerfileFilePath)
-          .Select(line => FromLinePattern.Match(line))
-          .Where(match => match.Success)
-          .Select(match => match.Groups["image"])
-          .Select(group => group.Value)
-          .Select(image => new DockerImage(image))
-          .Select(image => PullImageAsync(image, ct)));
-      }
-
       if (configuration.ImageBuildPolicy(cachedImage))
       {
-        _ = await Image.BuildAsync(configuration, ct)
+        var dockerfileArchive = new DockerfileArchive(configuration.DockerfileDirectory, configuration.Dockerfile, configuration.Image, _logger);
+
+        await Task.WhenAll(dockerfileArchive.GetBaseImages().Select(image => PullImageAsync(image, ct)))
+          .ConfigureAwait(false);
+
+        _ = await Image.BuildAsync(configuration, dockerfileArchive, ct)
           .ConfigureAwait(false);
       }
 

src/Testcontainers/Configurations/WaitStrategies/IWaitForContainerOS.cs

@@ -103,7 +103,7 @@ public interface IWaitForContainerOS
     /// <summary>
     /// Returns a collection with all configured wait strategies.
     /// </summary>
-    /// <returns>List with all configured wait strategies.</returns>
+    /// <returns>Returns a list with all configured wait strategies.</returns>
     [PublicAPI]
     IEnumerable<IWaitUntil> Build();
   }

src/Testcontainers/Images/DockerfileArchive.cs

@@ -17,6 +17,8 @@ namespace DotNet.Testcontainers.Images
   /// </summary>
   internal sealed class DockerfileArchive : ITarArchive
   {
+    private static readonly Regex FromLinePattern = new Regex("FROM (?<arg>--\\S+\\s)*(?<image>\\S+).*", RegexOptions.None, TimeSpan.FromSeconds(1));
+
     private readonly DirectoryInfo _dockerfileDirectory;
 
     private readonly FileInfo _dockerfile;
@@ -64,6 +66,54 @@ public DockerfileArchive(DirectoryInfo dockerfileDirectory, FileInfo dockerfile,
       _logger = logger;
     }
 
+    /// <summary>
+    /// Gets a collection of base images.
+    /// </summary>
+    /// <remarks>
+    /// This method reads the Dockerfile and collects a list of base images. It
+    /// excludes stages that do not correspond to base images. For example, it will not include
+    /// the second line from the following Dockerfile configuration:
+    /// <code>
+    ///   FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build
+    ///   FROM build
+    /// </code>
+    /// </remarks>
+    /// <returns>An <see cref="IEnumerable{T}" /> of <see cref="IImage" />.</returns>
+    public IEnumerable<IImage> GetBaseImages()
+    {
+      const string imageGroup = "image";
+
+      var lines = File.ReadAllLines(Path.Combine(_dockerfileDirectory.FullName, _dockerfile.ToString()))
+        .Select(line => line.Trim())
+        .Where(line => !string.IsNullOrEmpty(line))
+        .Where(line => !line.StartsWith("#", StringComparison.Ordinal))
+        .Select(line => FromLinePattern.Match(line))
+        .Where(match => match.Success)
+        // Until now, we are unable to resolve variables within Dockerfiles. Ignore base
+        // images that utilize variables. Expect them to exist on the host.
+        .Where(match => !match.Groups[imageGroup].Value.Contains('$'))
+        .Where(match => !match.Groups[imageGroup].Value.Any(char.IsUpper))
+        .ToArray();
+
+      var stages = lines
+        .Select(line => line.Value)
+        .Select(line => line.Split(new [] { " AS ", " As ", " aS ", " as " }, StringSplitOptions.RemoveEmptyEntries))
+        .Where(substrings => substrings.Length > 1)
+        .Select(substrings => substrings[substrings.Length - 1])
+        .Distinct()
+        .ToArray();
+
+      var images = lines
+        .Select(match => match.Groups[imageGroup])
+        .Select(group => group.Value)
+        .Where(value => !stages.Contains(value))
+        .Distinct()
+        .Select(value => new DockerImage(value))
+        .ToArray();
+
+      return images;
+    }
+
     /// <inheritdoc />
     public async Task<string> Tar(CancellationToken ct = default)
     {

tests/Testcontainers.Tests/Assets/.dockerignore

@@ -2,4 +2,5 @@ Dockerfile
 credHelpers
 credsStore
 healthWaitStrategy
+pullBaseImages
 **/*.md

tests/Testcontainers.Tests/Assets/pullBaseImages/Dockerfile

@@ -0,0 +1,8 @@
+ARG REPO=mcr.microsoft.com/dotnet/aspnet
+FROM $REPO:6.0.21-jammy-amd64
+FROM ${REPO}:6.0.21-jammy-amd64
+FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build
+FROM mcr.microsoft.com/dotnet/runtime:6.0 AS runtime
+FROM build
+FROM build AS publish
+FROM mcr.microsoft.com/dotnet/aspnet:6.0.21-jammy-amd64

tests/Testcontainers.Tests/Unit/Images/ImageFromDockerfileTest.cs

@@ -3,6 +3,7 @@ namespace DotNet.Testcontainers.Tests.Unit
   using System;
   using System.Collections.Generic;
   using System.IO;
+  using System.Linq;
   using System.Text;
   using System.Threading.Tasks;
   using DotNet.Testcontainers.Builders;
@@ -14,17 +15,32 @@ namespace DotNet.Testcontainers.Tests.Unit
 
   public sealed class ImageFromDockerfileTest
   {
+    [Fact]
+    public void DockerfileArchiveGetBaseImages()
+    {
+      // Given
+      IImage image = new DockerImage("localhost/testcontainers", Guid.NewGuid().ToString("D"), string.Empty);
+
+      var dockerfileArchive = new DockerfileArchive("Assets//pullBaseImages/", "Dockerfile", image, NullLogger.Instance);
+
+      // When
+      var baseImages = dockerfileArchive.GetBaseImages();
+
+      // Then
+      Assert.Equal(3, baseImages.Count());
+    }
+
     [Fact]
     public async Task DockerfileArchiveTar()
     {
       // Given
-      var image = new DockerImage("testcontainers", "test", "0.1.0");
+      IImage image = new DockerImage("localhost/testcontainers", Guid.NewGuid().ToString("D"), string.Empty);
 
       var expected = new SortedSet<string> { ".dockerignore", "Dockerfile", "setup/setup.sh" };
 
       var actual = new SortedSet<string>();
 
-      var dockerfileArchive = new DockerfileArchive("Assets", "Dockerfile", image, NullLogger.Instance);
+      var dockerfileArchive = new DockerfileArchive("Assets/", "Dockerfile", image, NullLogger.Instance);
 
       var dockerfileArchiveFilePath = await dockerfileArchive.Tar()
         .ConfigureAwait(false);
@@ -91,7 +107,7 @@ public async Task BuildsDockerImage()
       var imageFromDockerfileBuilder = new ImageFromDockerfileBuilder()
         .WithName(tag1)
         .WithDockerfile("Dockerfile")
-        .WithDockerfileDirectory("Assets")
+        .WithDockerfileDirectory("Assets/")
         .WithDeleteIfExists(true)
         .WithCreateParameterModifier(parameterModifier => parameterModifier.Tags.Add(tag2.FullName))
         .Build();