From 0c5ce32293fc0421089ed1517eee728a3d6e1e59 Mon Sep 17 00:00:00 2001 From: Andre Hofmeister <9199345+HofmeisterAn@users.noreply.github.com> Date: Mon, 13 Nov 2023 20:11:06 +0100 Subject: [PATCH] feat: Improve tar stream logging (copy files to container) --- .../Builders/ContainerBuilder`3.cs | 2 +- .../Clients/DockerContainerOperations.cs | 4 +- .../Clients/IDockerContainerOperations.cs | 2 +- .../Clients/TestcontainersClient.cs | 8 +- .../Containers/ContainerConfiguration.cs | 6 +- .../Containers/IContainerConfiguration.cs | 4 +- .../Containers/TarOutputMemoryStream.cs | 31 +++++++- src/Testcontainers/Logging.cs | 8 +- .../TarOutputMemoryStreamTest.cs | 2 +- .../Usings.cs | 1 + .../Unix/CopyResourceMappingContainerTest.cs | 75 ------------------- 11 files changed, 47 insertions(+), 96 deletions(-) delete mode 100644 tests/Testcontainers.Tests/Unit/Containers/Unix/CopyResourceMappingContainerTest.cs diff --git a/src/Testcontainers/Builders/ContainerBuilder`3.cs b/src/Testcontainers/Builders/ContainerBuilder`3.cs index 6df73eb1f..ef68e607c 100644 --- a/src/Testcontainers/Builders/ContainerBuilder`3.cs +++ b/src/Testcontainers/Builders/ContainerBuilder`3.cs @@ -184,7 +184,7 @@ public TBuilderEntity WithPortBinding(string hostPort, string containerPort) /// public TBuilderEntity WithResourceMapping(IResourceMapping resourceMapping) { - var resourceMappings = new Dictionary { { resourceMapping.Target, resourceMapping } }; + var resourceMappings = new[] { resourceMapping }; return Clone(new ContainerConfiguration(resourceMappings: resourceMappings)); } diff --git a/src/Testcontainers/Clients/DockerContainerOperations.cs b/src/Testcontainers/Clients/DockerContainerOperations.cs index aee7e39b1..42cd9d9c0 100644 --- a/src/Testcontainers/Clients/DockerContainerOperations.cs +++ b/src/Testcontainers/Clients/DockerContainerOperations.cs @@ -100,9 +100,9 @@ public Task RemoveAsync(string id, CancellationToken ct = default) return Docker.Containers.RemoveContainerAsync(id, new ContainerRemoveParameters { Force = true, RemoveVolumes = true }, ct); } - public Task ExtractArchiveToContainerAsync(string id, string path, Stream tarStream, CancellationToken ct = default) + public Task ExtractArchiveToContainerAsync(string id, string path, TarOutputMemoryStream tarStream, CancellationToken ct = default) { - _logger.CopyArchiveToDockerContainer(id, path); + _logger.CopyArchiveToDockerContainer(id, tarStream.ContentLength); return Docker.Containers.ExtractArchiveToContainerAsync(id, new ContainerPathStatParameters { Path = path, AllowOverwriteDirWithFile = false }, tarStream, ct); } diff --git a/src/Testcontainers/Clients/IDockerContainerOperations.cs b/src/Testcontainers/Clients/IDockerContainerOperations.cs index 1557074a7..2b942c05a 100644 --- a/src/Testcontainers/Clients/IDockerContainerOperations.cs +++ b/src/Testcontainers/Clients/IDockerContainerOperations.cs @@ -21,7 +21,7 @@ internal interface IDockerContainerOperations : IHasListOperations GetArchiveFromContainerAsync(string id, string path, CancellationToken ct = default); diff --git a/src/Testcontainers/Clients/TestcontainersClient.cs b/src/Testcontainers/Clients/TestcontainersClient.cs index 2714bb0a6..d7b397c1b 100644 --- a/src/Testcontainers/Clients/TestcontainersClient.cs +++ b/src/Testcontainers/Clients/TestcontainersClient.cs @@ -193,7 +193,7 @@ public async Task CopyAsync(string id, IResourceMapping resourceMapping, Cancell return; } - using (var tarOutputMemStream = new TarOutputMemoryStream()) + using (var tarOutputMemStream = new TarOutputMemoryStream(_logger)) { await tarOutputMemStream.AddAsync(resourceMapping, ct) .ConfigureAwait(false); @@ -209,7 +209,7 @@ await Container.ExtractArchiveToContainerAsync(id, "/", tarOutputMemStream, ct) /// public async Task CopyAsync(string id, DirectoryInfo source, string target, UnixFileModes fileMode, CancellationToken ct = default) { - using (var tarOutputMemStream = new TarOutputMemoryStream(target)) + using (var tarOutputMemStream = new TarOutputMemoryStream(target, _logger)) { await tarOutputMemStream.AddAsync(source, true, fileMode, ct) .ConfigureAwait(false); @@ -225,7 +225,7 @@ await Container.ExtractArchiveToContainerAsync(id, "/", tarOutputMemStream, ct) /// public async Task CopyAsync(string id, FileInfo source, string target, UnixFileModes fileMode, CancellationToken ct = default) { - using (var tarOutputMemStream = new TarOutputMemoryStream(target)) + using (var tarOutputMemStream = new TarOutputMemoryStream(target, _logger)) { await tarOutputMemStream.AddAsync(source, fileMode, ct) .ConfigureAwait(false); @@ -313,7 +313,7 @@ await Network.ConnectAsync("bridge", id, ct) if (configuration.ResourceMappings.Any()) { - await Task.WhenAll(configuration.ResourceMappings.Values.Select(resourceMapping => CopyAsync(id, resourceMapping, ct))) + await Task.WhenAll(configuration.ResourceMappings.Select(resourceMapping => CopyAsync(id, resourceMapping, ct))) .ConfigureAwait(false); } diff --git a/src/Testcontainers/Configurations/Containers/ContainerConfiguration.cs b/src/Testcontainers/Configurations/Containers/ContainerConfiguration.cs index 0a4b6092d..34833b3b7 100644 --- a/src/Testcontainers/Configurations/Containers/ContainerConfiguration.cs +++ b/src/Testcontainers/Configurations/Containers/ContainerConfiguration.cs @@ -29,7 +29,7 @@ public class ContainerConfiguration : ResourceConfigurationA dictionary of environment variables. /// A dictionary of exposed ports. /// A dictionary of port bindings. - /// A dictionary of resource mappings. + /// A list of resource mappings. /// A list of containers. /// A list of mounts. /// A list of networks. @@ -52,7 +52,7 @@ public ContainerConfiguration( IReadOnlyDictionary environments = null, IReadOnlyDictionary exposedPorts = null, IReadOnlyDictionary portBindings = null, - IReadOnlyDictionary resourceMappings = null, + IEnumerable resourceMappings = null, IEnumerable containers = null, IEnumerable mounts = null, IEnumerable networks = null, @@ -178,7 +178,7 @@ public ContainerConfiguration(IContainerConfiguration oldValue, IContainerConfig public IReadOnlyDictionary PortBindings { get; } /// - public IReadOnlyDictionary ResourceMappings { get; } + public IEnumerable ResourceMappings { get; } /// public IEnumerable Containers { get; } diff --git a/src/Testcontainers/Configurations/Containers/IContainerConfiguration.cs b/src/Testcontainers/Configurations/Containers/IContainerConfiguration.cs index 8c10cb360..1093e8480 100644 --- a/src/Testcontainers/Configurations/Containers/IContainerConfiguration.cs +++ b/src/Testcontainers/Configurations/Containers/IContainerConfiguration.cs @@ -82,9 +82,9 @@ public interface IContainerConfiguration : IResourceConfiguration PortBindings { get; } /// - /// Gets a dictionary of resource mappings. + /// Gets a list of resource mappings. /// - IReadOnlyDictionary ResourceMappings { get; } + IEnumerable ResourceMappings { get; } /// /// Gets a list of dependent containers. diff --git a/src/Testcontainers/Containers/TarOutputMemoryStream.cs b/src/Testcontainers/Containers/TarOutputMemoryStream.cs index 8b809ba90..245362f98 100644 --- a/src/Testcontainers/Containers/TarOutputMemoryStream.cs +++ b/src/Testcontainers/Containers/TarOutputMemoryStream.cs @@ -7,6 +7,7 @@ namespace DotNet.Testcontainers.Containers using System.Threading.Tasks; using DotNet.Testcontainers.Configurations; using ICSharpCode.SharpZipLib.Tar; + using Microsoft.Extensions.Logging; /// /// Represent a tar archive file. @@ -15,12 +16,17 @@ public sealed class TarOutputMemoryStream : TarOutputStream { private readonly string _targetDirectoryPath; + private readonly ILogger _logger; + + private long _contentLength; + /// /// Initializes a new instance of the class. /// /// The target directory path to extract the files to. - public TarOutputMemoryStream(string targetDirectoryPath) - : this() + /// The logger. + public TarOutputMemoryStream(string targetDirectoryPath, ILogger logger) + : this(logger) { _targetDirectoryPath = targetDirectoryPath; } @@ -28,12 +34,23 @@ public TarOutputMemoryStream(string targetDirectoryPath) /// /// Initializes a new instance of the class. /// - public TarOutputMemoryStream() + /// The logger. + public TarOutputMemoryStream(ILogger logger) : base(new MemoryStream(), Encoding.Default) { + _logger = logger; IsStreamOwner = false; } + /// + /// Gets the content length. + /// + /// + /// The initial tar output stream length is 10240 bytes (SharpZipLib). The stream + /// length does not correspond to the actual content's length. + /// + public long ContentLength => _contentLength; + /// /// Adds the content of an implementation of to the archive. /// @@ -52,6 +69,8 @@ public async Task AddAsync(IResourceMapping resourceMapping, CancellationToken c tarEntry.TarHeader.ModTime = DateTime.UtcNow; tarEntry.Size = fileContent.Length; + _logger.LogInformation("Add file to tar archive: Content length: {Length} byte(s), Target file: \"{Target}\"", tarEntry.Size, targetFilePath); + await PutNextEntryAsync(tarEntry, ct) .ConfigureAwait(false); @@ -65,6 +84,8 @@ await WriteAsync(fileContent, 0, fileContent.Length, ct) await CloseEntryAsync(ct) .ConfigureAwait(false); + + _ = Interlocked.Add(ref _contentLength, tarEntry.Size); } /// @@ -116,6 +137,8 @@ public async Task AddAsync(DirectoryInfo directory, FileInfo file, UnixFileModes tarEntry.TarHeader.ModTime = file.LastWriteTimeUtc; tarEntry.Size = stream.Length; + _logger.LogInformation("Add file to tar archive: Source file: \"{Source}\", Target file: \"{Target}\"", tarEntry.TarHeader.Name, targetFilePath); + await PutNextEntryAsync(tarEntry, ct) .ConfigureAwait(false); @@ -124,6 +147,8 @@ await stream.CopyToAsync(this, 81920, ct) await CloseEntryAsync(ct) .ConfigureAwait(false); + + _ = Interlocked.Add(ref _contentLength, tarEntry.Size); } } } diff --git a/src/Testcontainers/Logging.cs b/src/Testcontainers/Logging.cs index 12318b354..c9bfddf90 100644 --- a/src/Testcontainers/Logging.cs +++ b/src/Testcontainers/Logging.cs @@ -31,8 +31,8 @@ private static readonly Action _StartReadinessCheck private static readonly Action _CompleteReadinessCheck = LoggerMessage.Define(LogLevel.Information, default, "Docker container {Id} ready"); - private static readonly Action _CopyArchiveToDockerContainer - = LoggerMessage.Define(LogLevel.Information, default, "Copy tar archive to \"{Path}\" to Docker container {Id}"); + private static readonly Action _CopyArchiveToDockerContainer + = LoggerMessage.Define(LogLevel.Information, default, "Copy tar archive to container: Content length: {Length} byte(s), Docker container: {Id}"); private static readonly Action _ReadArchiveFromDockerContainer = LoggerMessage.Define(LogLevel.Information, default, "Read \"{Path}\" from Docker container {Id}"); @@ -125,9 +125,9 @@ public static void CompleteReadinessCheck(this ILogger logger, string id) _CompleteReadinessCheck(logger, TruncId(id), null); } - public static void CopyArchiveToDockerContainer(this ILogger logger, string id, string path) + public static void CopyArchiveToDockerContainer(this ILogger logger, string id, long length) { - _CopyArchiveToDockerContainer(logger, path, TruncId(id), null); + _CopyArchiveToDockerContainer(logger, length, TruncId(id), null); } public static void ReadArchiveFromDockerContainer(this ILogger logger, string id, string path) diff --git a/tests/Testcontainers.Platform.Linux.Tests/TarOutputMemoryStreamTest.cs b/tests/Testcontainers.Platform.Linux.Tests/TarOutputMemoryStreamTest.cs index 1f89c5010..d27090463 100644 --- a/tests/Testcontainers.Platform.Linux.Tests/TarOutputMemoryStreamTest.cs +++ b/tests/Testcontainers.Platform.Linux.Tests/TarOutputMemoryStreamTest.cs @@ -4,7 +4,7 @@ public abstract class TarOutputMemoryStreamTest { private const string TargetDirectoryPath = "/tmp"; - private readonly TarOutputMemoryStream _tarOutputMemoryStream = new TarOutputMemoryStream(TargetDirectoryPath); + private readonly TarOutputMemoryStream _tarOutputMemoryStream = new TarOutputMemoryStream(TargetDirectoryPath, NullLogger.Instance); private readonly FileInfo _testFile = new FileInfo(Path.Combine(TestSession.TempDirectoryPath, Path.GetRandomFileName())); diff --git a/tests/Testcontainers.Platform.Linux.Tests/Usings.cs b/tests/Testcontainers.Platform.Linux.Tests/Usings.cs index fb155c2b8..69f9a7f8a 100644 --- a/tests/Testcontainers.Platform.Linux.Tests/Usings.cs +++ b/tests/Testcontainers.Platform.Linux.Tests/Usings.cs @@ -16,4 +16,5 @@ global using DotNet.Testcontainers.Containers; global using ICSharpCode.SharpZipLib.Tar; global using JetBrains.Annotations; +global using Microsoft.Extensions.Logging.Abstractions; global using Xunit; \ No newline at end of file diff --git a/tests/Testcontainers.Tests/Unit/Containers/Unix/CopyResourceMappingContainerTest.cs b/tests/Testcontainers.Tests/Unit/Containers/Unix/CopyResourceMappingContainerTest.cs deleted file mode 100644 index eeb85acc2..000000000 --- a/tests/Testcontainers.Tests/Unit/Containers/Unix/CopyResourceMappingContainerTest.cs +++ /dev/null @@ -1,75 +0,0 @@ -namespace DotNet.Testcontainers.Tests.Unit -{ - using System; - using System.Collections.Generic; - using System.IO; - using System.Linq; - using System.Text; - using System.Threading.Tasks; - using DotNet.Testcontainers.Builders; - using DotNet.Testcontainers.Commons; - using DotNet.Testcontainers.Containers; - using Xunit; - - public sealed class CopyResourceMappingContainerTest : IAsyncLifetime, IDisposable - { - private const string ResourceMappingContent = "👋"; - - private readonly FileInfo _testFile = new FileInfo(Path.Combine(TestSession.TempDirectoryPath, Path.GetRandomFileName())); - - private readonly string _bytesTargetFilePath; - - private readonly string _fileTargetFilePath; - - private readonly IContainer _container; - - public CopyResourceMappingContainerTest() - { - var resourceContent = Encoding.Default.GetBytes(ResourceMappingContent); - - using var fileStream = _testFile.Open(FileMode.Create, FileAccess.Write, FileShare.ReadWrite); - fileStream.Write(resourceContent); - - _bytesTargetFilePath = string.Join("/", string.Empty, "tmp", Guid.NewGuid(), _testFile.Name); - - _fileTargetFilePath = string.Join("/", string.Empty, "tmp", Guid.NewGuid()); - - _container = new ContainerBuilder() - .WithImage(CommonImages.Alpine) - .WithResourceMapping(resourceContent, _bytesTargetFilePath) - .WithResourceMapping(_testFile, _fileTargetFilePath) - .Build(); - } - - public Task InitializeAsync() - { - return _container.StartAsync(); - } - - public Task DisposeAsync() - { - return _container.DisposeAsync().AsTask(); - } - - public void Dispose() - { - _testFile.Delete(); - } - - [Fact] - public async Task ReadExistingFile() - { - // Given - IList targetFilePaths = new List(); - targetFilePaths.Add(_bytesTargetFilePath); - targetFilePaths.Add(string.Join("/", _fileTargetFilePath, _testFile.Name)); - - // When - var resourceContents = await Task.WhenAll(targetFilePaths.Select(containerFilePath => _container.ReadFileAsync(containerFilePath))) - .ConfigureAwait(false); - - // Then - Assert.All(resourceContents.Select(Encoding.Default.GetString), resourceContent => Assert.Equal(ResourceMappingContent, resourceContent)); - } - } -}