Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add Azure EventHubs module #1183

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
<PackageVersion Include="AWSSDK.SimpleNotificationService" Version="3.7.101.7"/>
<PackageVersion Include="AWSSDK.SQS" Version="3.7.100.71"/>
<PackageVersion Include="Azure.Data.Tables" Version="12.8.0"/>
<PackageVersion Include="Azure.Messaging.EventHubs" Version="5.11.3" />
<PackageVersion Include="Azure.Storage.Blobs" Version="12.17.0"/>
<PackageVersion Include="Azure.Storage.Queues" Version="12.15.0"/>
<PackageVersion Include="ClickHouse.Client" Version="6.7.1"/>
Expand Down
14 changes: 14 additions & 0 deletions Testcontainers.sln
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Testcontainers.Tests", "tes
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Testcontainers.WebDriver.Tests", "tests\Testcontainers.WebDriver.Tests\Testcontainers.WebDriver.Tests.csproj", "{EBA72C3B-57D5-43FF-A5B4-3D55B3B6D4C2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Testcontainers.EventHubs", "src\Testcontainers.EventHubs\Testcontainers.EventHubs.csproj", "{0EF885E9-E973-47DC-AA9C-3A5E9175B0F3}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Testcontainers.EventHubs.Tests", "tests\Testcontainers.EventHubs.Tests\Testcontainers.EventHubs.Tests.csproj", "{4A0C5523-CEB2-49C9-AE62-9187A01B016B}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -568,6 +572,14 @@ Global
{EBA72C3B-57D5-43FF-A5B4-3D55B3B6D4C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EBA72C3B-57D5-43FF-A5B4-3D55B3B6D4C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{EBA72C3B-57D5-43FF-A5B4-3D55B3B6D4C2}.Release|Any CPU.Build.0 = Release|Any CPU
{0EF885E9-E973-47DC-AA9C-3A5E9175B0F3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0EF885E9-E973-47DC-AA9C-3A5E9175B0F3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0EF885E9-E973-47DC-AA9C-3A5E9175B0F3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0EF885E9-E973-47DC-AA9C-3A5E9175B0F3}.Release|Any CPU.Build.0 = Release|Any CPU
{4A0C5523-CEB2-49C9-AE62-9187A01B016B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4A0C5523-CEB2-49C9-AE62-9187A01B016B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4A0C5523-CEB2-49C9-AE62-9187A01B016B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4A0C5523-CEB2-49C9-AE62-9187A01B016B}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{5365F780-0E6C-41F0-B1B9-7DC34368F80C} = {673F23AE-7694-4BB9-ABD4-136D6C13634E}
Expand Down Expand Up @@ -661,5 +673,7 @@ Global
{1A1983E6-5297-435F-B467-E8E1F11277D6} = {7164F1FB-7F24-444A-ACD2-2C329C2B3CCF}
{27CDB869-A150-4593-958F-6F26E5391E7C} = {7164F1FB-7F24-444A-ACD2-2C329C2B3CCF}
{EBA72C3B-57D5-43FF-A5B4-3D55B3B6D4C2} = {7164F1FB-7F24-444A-ACD2-2C329C2B3CCF}
{0EF885E9-E973-47DC-AA9C-3A5E9175B0F3} = {673F23AE-7694-4BB9-ABD4-136D6C13634E}
{4A0C5523-CEB2-49C9-AE62-9187A01B016B} = {7164F1FB-7F24-444A-ACD2-2C329C2B3CCF}
EndGlobalSection
EndGlobal
1 change: 1 addition & 0 deletions src/Testcontainers.EventHubs/.editorconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
root = true
53 changes: 53 additions & 0 deletions src/Testcontainers.EventHubs/Configuration/ConfigurationBuilder.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
namespace Testcontainers.EventHubs.Configuration
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please change all to file scoped namespaces.

{
public class ConfigurationBuilder
{
private const string DefaultNamespace = "emulatorns1";

private readonly RootConfiguration _rootConfiguration = new RootConfiguration();

private ConfigurationBuilder()
{
_rootConfiguration.UserConfig = new UserConfig
{
NamespaceConfig = new List<NamespaceConfig>()
{
new NamespaceConfig
{
Type = "EventHub",
Name = DefaultNamespace
}
},
LoggingConfig = new LoggingConfig() { Type = "File" }
};
}

public static ConfigurationBuilder Create()
{
return new ConfigurationBuilder();
}

public ConfigurationBuilder WithEventHub(string entityName, string partitionCount, IEnumerable<string> consumerGroups)
{
var namespaceConfig = _rootConfiguration.UserConfig.NamespaceConfig.FirstOrDefault(x => x.Name == DefaultNamespace);
if (namespaceConfig == null)
{
throw new InvalidOperationException($"Default Namespace '{DefaultNamespace}' not found.");
}

namespaceConfig.Entities.Add(new Entity
{
Name = entityName,
PartitionCount = partitionCount,
ConsumerGroups = consumerGroups.Select(consumerGroupName => new ConsumerGroup { Name = consumerGroupName }).ToList()
});

return this;
}

public string Build()
{
return JsonSerializer.Serialize(_rootConfiguration);
}
}
}
7 changes: 7 additions & 0 deletions src/Testcontainers.EventHubs/Configuration/ConsumerGroup.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespace Testcontainers.EventHubs.Configuration
{
public record ConsumerGroup
{
public string Name { get; set; }
}
}
9 changes: 9 additions & 0 deletions src/Testcontainers.EventHubs/Configuration/Entity.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
namespace Testcontainers.EventHubs.Configuration
{
public record Entity
{
public string Name { get; set; }
public string PartitionCount { get; set; }
public List<ConsumerGroup> ConsumerGroups { get; set; } = [];
}
}
7 changes: 7 additions & 0 deletions src/Testcontainers.EventHubs/Configuration/LoggingConfig.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespace Testcontainers.EventHubs.Configuration
{
public record LoggingConfig
{
public string Type { get; set; }
}
}
10 changes: 10 additions & 0 deletions src/Testcontainers.EventHubs/Configuration/NamespaceConfig.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
namespace Testcontainers.EventHubs.Configuration
{
public record NamespaceConfig
{
public string Type { get; set; }
public string Name { get; set; }

public List<Entity> Entities { get; set; } = [];
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespace Testcontainers.EventHubs.Configuration
{
public record RootConfiguration
{
public UserConfig UserConfig { get; set; }
}
}
8 changes: 8 additions & 0 deletions src/Testcontainers.EventHubs/Configuration/UserConfig.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
namespace Testcontainers.EventHubs.Configuration
{
public record UserConfig
{
public List<NamespaceConfig> NamespaceConfig { get; set; } = [];
public LoggingConfig LoggingConfig { get; set; }
}
}
127 changes: 127 additions & 0 deletions src/Testcontainers.EventHubs/EventHubsBuilder.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
using DotNet.Testcontainers;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use the global usings file.


namespace Testcontainers.EventHubs;

/// <inheritdoc cref="ContainerBuilder{TBuilderEntity, TContainerEntity, TConfigurationEntity}" />
[PublicAPI]
public sealed class EventHubsBuilder : ContainerBuilder<EventHubsBuilder, EventHubsContainer, EventHubsConfiguration>
{
public const string EventHubsImage = "mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest";

public const ushort EventHubsPort = 5672;

/// <summary>
/// Initializes a new instance of the <see cref="EventHubsBuilder" /> class.
/// </summary>
public EventHubsBuilder()
: this(new EventHubsConfiguration())
{
DockerResourceConfiguration = Init().DockerResourceConfiguration;
}

/// <summary>
/// Initializes a new instance of the <see cref="EventHubsBuilder" /> class.
/// </summary>
/// <param name="resourceConfiguration">The Docker resource configuration.</param>
private EventHubsBuilder(EventHubsConfiguration resourceConfiguration)
: base(resourceConfiguration)
{
DockerResourceConfiguration = resourceConfiguration;
}

/// <inheritdoc />
protected override EventHubsConfiguration DockerResourceConfiguration { get; }

/// <summary>
/// Sets the event hub configuration
/// </summary>
/// <param name="configurationBuilder"></param>
/// <returns></returns>
public EventHubsBuilder WithConfigurationBuilder(ConfigurationBuilder configurationBuilder)
{
var configBytes = Encoding.UTF8.GetBytes(configurationBuilder.Build());

return Merge(DockerResourceConfiguration, new EventHubsConfiguration(configurationBuilder: configurationBuilder))
.WithResourceMapping(configBytes, "Eventhubs_Emulator/ConfigFiles/Config.json");
}
Comment on lines +40 to +46
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this builder cover all Event Hubs configuration? Are we responsible for it? Is there a default configuration? The EventHubsBuilder will fail if the user does not call WithConfigurationBuilder(ConfigurationBuilder) explicitly. Until now, no other module requires an additional explicit call to a builder method (expect license agreements). _ = new FooBuilder.Build() always returns a working configuration. This is something I favor.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With configuration builder is needed, because we don't have a default eventhub assigned to the default namespace. We can eventually introduce something, but I don't advise to do so as usually we want to have something custom. For an example, we have azure function:

public static class ProcessSalesData
{
    [FunctionName("ProcessSalesData")]
    public static void Run(
        [EventHubTrigger("SalesDataHub", Connection = "EventHubConnectionAppSetting", ConsumerGroup = "RealTimeAnalytics")] string[] events,
        ILogger log)
    {
        foreach (var eventData in events)
        {
            log.LogInformation($"C# Event Hub trigger function processed an event: {eventData}");
        }
    }
}

Our eventhub is SalesDataHub and usually in integration tests we won't enforce to change it to the ones that is created by the testcontainer.eventhubs. We'd rather to replicate solution that we have in our infra with custom event hub names and just automatically subscribe to this when new message came up.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On a second thought we can introduce eh1 by default. Always we will be able to expand that even further, but for most of the time maybe that will be enought (for most scenarios).


/// <summary>
/// Sets the endpoint of the azurite blob service
/// </summary>
/// <param name="azuriteBlobEndpoint"></param>
/// <returns></returns>
public EventHubsBuilder WithAzuriteBlobEndpoint(string azuriteBlobEndpoint)
{
return Merge(DockerResourceConfiguration, new EventHubsConfiguration(azuriteBlobEndpoint: azuriteBlobEndpoint))
.WithEnvironment("BLOB_SERVER", azuriteBlobEndpoint);
}

/// <summary>
/// Sets the endpoint of the azurite table service
/// </summary>
/// <param name="azuriteTableEndpoint"></param>
/// <returns></returns>
public EventHubsBuilder WithAzuriteTableEndpoint(string azuriteTableEndpoint)
{
return Merge(DockerResourceConfiguration, new EventHubsConfiguration(azuriteTableEndpoint: azuriteTableEndpoint))
.WithEnvironment("METADATA_SERVER", azuriteTableEndpoint);
}

/// <inheritdoc />
public override EventHubsContainer Build()
{
Validate();

var waitStrategy = Wait.ForUnixContainer().UntilMessageIsLogged("Emulator Service is Successfully Up!");
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The wait strategy should be moved to Init().


var eventHubsBuilder = DockerResourceConfiguration.WaitStrategies.Count() > 1 ? this : WithWaitStrategy(waitStrategy);
return new EventHubsContainer(eventHubsBuilder.DockerResourceConfiguration);
}

/// <inheritdoc />
protected override void Validate()
{
base.Validate();

_ = Guard.Argument(DockerResourceConfiguration.ConfigurationBuilder,
nameof(DockerResourceConfiguration.ConfigurationBuilder))
.NotNull();

_ = Guard.Argument(DockerResourceConfiguration.AzuriteBlobEndpoint,
nameof(DockerResourceConfiguration.AzuriteBlobEndpoint))
.NotNull()
.NotEmpty();

_ = Guard.Argument(DockerResourceConfiguration.AzuriteTableEndpoint,
nameof(DockerResourceConfiguration.AzuriteTableEndpoint))
.NotNull()
.NotEmpty();
}

/// <inheritdoc />
protected override EventHubsBuilder Init()
{
return base.Init()
.WithImage(EventHubsImage)
.WithEnvironment("ACCEPT_EULA", "Y")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please do not hide license agreements. Right now, we use the following pattern.

.WithPortBinding(EventHubsPort, true);
}

/// <inheritdoc />
protected override EventHubsBuilder Clone(IResourceConfiguration<CreateContainerParameters> resourceConfiguration)
{
return Merge(DockerResourceConfiguration, new EventHubsConfiguration(resourceConfiguration));
}

/// <inheritdoc />
protected override EventHubsBuilder Clone(IContainerConfiguration resourceConfiguration)
{
return Merge(DockerResourceConfiguration, new EventHubsConfiguration(resourceConfiguration));
}

/// <inheritdoc />
protected override EventHubsBuilder Merge(EventHubsConfiguration oldValue, EventHubsConfiguration newValue)
{
return new EventHubsBuilder(new EventHubsConfiguration(oldValue, newValue));
}
}
80 changes: 80 additions & 0 deletions src/Testcontainers.EventHubs/EventHubsConfiguration.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
namespace Testcontainers.EventHubs;

/// <inheritdoc cref="ContainerConfiguration" />
[PublicAPI]
public sealed class EventHubsConfiguration : ContainerConfiguration
{
/// <summary>
/// Initializes a new instance of the <see cref="EventHubsConfiguration" /> class.
/// </summary>
/// <param name="configurationBuilder">The configuration builder.</param>
/// <param name="azuriteBlobEndpoint">The Azurite blob endpoint.</param>
/// <param name="azuriteTableEndpoint">The Azurite table endpoint.</param>
public EventHubsConfiguration(
ConfigurationBuilder configurationBuilder = null,
string azuriteBlobEndpoint = null,
string azuriteTableEndpoint = null)
{
ConfigurationBuilder = configurationBuilder;
AzuriteBlobEndpoint = azuriteBlobEndpoint;
AzuriteTableEndpoint = azuriteTableEndpoint;
}

/// <summary>
/// Initializes a new instance of the <see cref="EventHubsConfiguration" /> class.
/// </summary>
/// <param name="resourceConfiguration">The Docker resource configuration.</param>
public EventHubsConfiguration(IResourceConfiguration<CreateContainerParameters> resourceConfiguration)
: base(resourceConfiguration)
{
// Passes the configuration upwards to the base implementations to create an updated immutable copy.
}

/// <summary>
/// Initializes a new instance of the <see cref="EventHubsConfiguration" /> class.
/// </summary>
/// <param name="resourceConfiguration">The Docker resource configuration.</param>
public EventHubsConfiguration(IContainerConfiguration resourceConfiguration)
: base(resourceConfiguration)
{
// Passes the configuration upwards to the base implementations to create an updated immutable copy.
}

/// <summary>
/// Initializes a new instance of the <see cref="EventHubsConfiguration" /> class.
/// </summary>
/// <param name="resourceConfiguration">The Docker resource configuration.</param>
public EventHubsConfiguration(EventHubsConfiguration resourceConfiguration)
: this(new EventHubsConfiguration(), resourceConfiguration)
{
// Passes the configuration upwards to the base implementations to create an updated immutable copy.
}

/// <summary>
/// Initializes a new instance of the <see cref="EventHubsConfiguration" /> class.
/// </summary>
/// <param name="oldValue">The old Docker resource configuration.</param>
/// <param name="newValue">The new Docker resource configuration.</param>
public EventHubsConfiguration(EventHubsConfiguration oldValue, EventHubsConfiguration newValue)
: base(oldValue, newValue)
{
ConfigurationBuilder = BuildConfiguration.Combine(oldValue.ConfigurationBuilder, newValue.ConfigurationBuilder);
AzuriteBlobEndpoint = BuildConfiguration.Combine(oldValue.AzuriteBlobEndpoint, newValue.AzuriteBlobEndpoint);
AzuriteTableEndpoint = BuildConfiguration.Combine(oldValue.AzuriteTableEndpoint, newValue.AzuriteTableEndpoint);
}

/// <summary>
/// Gets the configuration builder
/// </summary>
public ConfigurationBuilder ConfigurationBuilder { get; }
Comment on lines +66 to +69
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like we do not use the ConfigurationBuilder sometime later. In that case, we do not need to store it in the EventHubsConfiguration.


/// <summary>
/// Gets the Azurite blob endpoint
/// </summary>
public string AzuriteBlobEndpoint { get; }

/// <summary>
/// Gets the Azurite table endpoint
/// </summary>
public string AzuriteTableEndpoint { get; }
}
Loading
Loading