From ab10cc18f7d31792033f3f2c9b0dad9255910614 Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 4 Jul 2024 08:53:50 +0200 Subject: [PATCH 01/80] FS clean up --- .../LogicalReplicationTest.cs | 96 ------------------- ...tgresOutboxPatternWithCDC.NET.Tests.csproj | 25 ----- .../Usings.cs | 1 - 3 files changed, 122 deletions(-) delete mode 100644 src/PostgresOutboxPatternWithCDC.NET.Tests/LogicalReplicationTest.cs delete mode 100644 src/PostgresOutboxPatternWithCDC.NET.Tests/PostgresOutboxPatternWithCDC.NET.Tests.csproj delete mode 100644 src/PostgresOutboxPatternWithCDC.NET.Tests/Usings.cs diff --git a/src/PostgresOutboxPatternWithCDC.NET.Tests/LogicalReplicationTest.cs b/src/PostgresOutboxPatternWithCDC.NET.Tests/LogicalReplicationTest.cs deleted file mode 100644 index 93195a2..0000000 --- a/src/PostgresOutboxPatternWithCDC.NET.Tests/LogicalReplicationTest.cs +++ /dev/null @@ -1,96 +0,0 @@ -using Commons.Events; -using PostgresOutbox.Events; -using PostgresOutbox.Serialization; -using PostgresOutbox.Subscriptions; -using PostgresOutbox.Subscriptions.Replication; -using Xunit.Abstractions; -using static PostgresOutbox.Subscriptions.Management.PublicationManagement; -using static PostgresOutbox.Subscriptions.Management.ReplicationSlotManagement; - -namespace PostgresOutboxPatternWithCDC.NET.Tests; - -public class LogicalReplicationTest -{ - private readonly ITestOutputHelper testOutputHelper; - - public LogicalReplicationTest(ITestOutputHelper testOutputHelper) - { - this.testOutputHelper = testOutputHelper; - } - - [Fact] - public async Task WALSubscriptionForNewEventsShouldWork() - { - var cancellationTokenSource = new CancellationTokenSource(); - var ct = cancellationTokenSource.Token; - - var eventsTable = await CreateEventsTable(ConnectrionString, ct); - - var subscriptionOptions = new SubscriptionOptions( - ConnectrionString, - new PublicationSetupOptions(Randomise("events_pub"),eventsTable), - new ReplicationSlotSetupOptions(Randomise("events_slot")), - new EventDataMapper() - ); - var subscription = new Subscription(); - - var events = subscription.Subscribe(subscriptionOptions, ct); - - var @event = new UserCreated(Guid.NewGuid(), Guid.NewGuid().ToString()); - await EventsAppender.AppendAsync(eventsTable, @event, ConnectrionString, ct); - - await foreach (var readEvent in events.WithCancellation(ct)) - { - testOutputHelper.WriteLine(JsonSerialization.ToJson(readEvent)); - Assert.Equal(@event, readEvent); - return; - } - } - - [Fact] - public async Task WALSubscriptionForOldEventsShouldWork() - { - var cancellationTokenSource = new CancellationTokenSource(); - var ct = cancellationTokenSource.Token; - - var eventsTable = await CreateEventsTable(ConnectrionString, ct); - - var subscriptionOptions = new SubscriptionOptions( - ConnectrionString, - new PublicationSetupOptions(Randomise("events_pub"),eventsTable), - new ReplicationSlotSetupOptions(Randomise("events_slot")), - new EventDataMapper() - ); - var subscription = new Subscription(); - - var @event = new UserCreated(Guid.NewGuid(), Guid.NewGuid().ToString()); - await EventsAppender.AppendAsync(eventsTable, @event, ConnectrionString, ct); - - var events = subscription.Subscribe(subscriptionOptions, ct); - - await foreach (var readEvent in events) - { - testOutputHelper.WriteLine(JsonSerialization.ToJson(readEvent)); - Assert.Equal(@event, readEvent); - return; - } - } - - private async Task CreateEventsTable( - string connectionString, - CancellationToken ct - ) - { - var tableName = Randomise("events"); - - await EventsTable.Create(connectionString, tableName, ct); - - return tableName; - } - - private static string Randomise(string prefix) => - $"{prefix}_{Guid.NewGuid().ToString().Replace("-", "")}"; - - private const string ConnectrionString = - "PORT = 5432; HOST = localhost; TIMEOUT = 15; POOLING = True; MINPOOLSIZE = 1; MAXPOOLSIZE = 100; COMMANDTIMEOUT = 20; DATABASE = 'postgres'; PASSWORD = 'Password12!'; USER ID = 'postgres'"; -} diff --git a/src/PostgresOutboxPatternWithCDC.NET.Tests/PostgresOutboxPatternWithCDC.NET.Tests.csproj b/src/PostgresOutboxPatternWithCDC.NET.Tests/PostgresOutboxPatternWithCDC.NET.Tests.csproj deleted file mode 100644 index 9e0d56c..0000000 --- a/src/PostgresOutboxPatternWithCDC.NET.Tests/PostgresOutboxPatternWithCDC.NET.Tests.csproj +++ /dev/null @@ -1,25 +0,0 @@ - - - - net8.0 - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - diff --git a/src/PostgresOutboxPatternWithCDC.NET.Tests/Usings.cs b/src/PostgresOutboxPatternWithCDC.NET.Tests/Usings.cs deleted file mode 100644 index 8c927eb..0000000 --- a/src/PostgresOutboxPatternWithCDC.NET.Tests/Usings.cs +++ /dev/null @@ -1 +0,0 @@ -global using Xunit; \ No newline at end of file From 977f2034cb7c1f8f9e42515a41c70ef647903cdb Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 4 Jul 2024 08:53:11 +0200 Subject: [PATCH 02/80] suppress CS1591 (Missing XML comment for publicly visible type or member) at project level --- src/Blumchen/Blumchen.csproj | 10 ++++++++++ src/Blumchen/Database/Run.cs | 2 -- src/Blumchen/MessageTableOptions.cs | 1 - src/Blumchen/Publications/MessageAppender.cs | 1 - .../Publications/PublisherSetupOptionsBuilder.cs | 1 - src/Blumchen/Serialization/IDictionaryExtensions.cs | 1 - src/Blumchen/Serialization/INamingPolicy.cs | 1 - src/Blumchen/Serialization/ITypeResolver.cs | 1 - src/Blumchen/Serialization/JsonSerialization.cs | 1 - src/Blumchen/Serialization/MessageUrnAttribute.cs | 1 - src/Blumchen/Subscriptions/IConsume.cs | 1 - .../Subscriptions/Management/PublicationManagement.cs | 1 - .../Management/ReplicationSlotManagement.cs | 1 - src/Blumchen/Subscriptions/MimeType.cs | 1 - .../Replication/IReplicationDataMapper.cs | 1 - .../ReplicationMessageHandlers/Envelope.cs | 1 - .../Subscriptions/SnapshotReader/SnapshotReader.cs | 1 - src/Blumchen/Subscriptions/Subscription.cs | 1 - .../Subscriptions/SubscriptionOptionsBuilder.cs | 2 +- 19 files changed, 11 insertions(+), 19 deletions(-) diff --git a/src/Blumchen/Blumchen.csproj b/src/Blumchen/Blumchen.csproj index 4e6b52c..0f26919 100644 --- a/src/Blumchen/Blumchen.csproj +++ b/src/Blumchen/Blumchen.csproj @@ -25,6 +25,16 @@ snupkg Blumchen + + + 1591 + + + + + 1591 + + <_Parameter1>Tests diff --git a/src/Blumchen/Database/Run.cs b/src/Blumchen/Database/Run.cs index 4e03b17..f4ccec1 100644 --- a/src/Blumchen/Database/Run.cs +++ b/src/Blumchen/Database/Run.cs @@ -4,8 +4,6 @@ using Blumchen.Subscriptions.ReplicationMessageHandlers; using Npgsql; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member - namespace Blumchen.Database; public static class Run diff --git a/src/Blumchen/MessageTableOptions.cs b/src/Blumchen/MessageTableOptions.cs index a7fe0b1..7216108 100644 --- a/src/Blumchen/MessageTableOptions.cs +++ b/src/Blumchen/MessageTableOptions.cs @@ -3,7 +3,6 @@ namespace Blumchen; -#pragma warning disable CS1591 public record TableDescriptorBuilder { private MessageTable TableDescriptor { get; set; } = new(); diff --git a/src/Blumchen/Publications/MessageAppender.cs b/src/Blumchen/Publications/MessageAppender.cs index 623c8b4..c927dfe 100644 --- a/src/Blumchen/Publications/MessageAppender.cs +++ b/src/Blumchen/Publications/MessageAppender.cs @@ -3,7 +3,6 @@ using Npgsql; namespace Blumchen.Publications; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public static class MessageAppender { diff --git a/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs b/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs index 90e7792..dbd7095 100644 --- a/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs +++ b/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs @@ -5,7 +5,6 @@ namespace Blumchen.Publications; -#pragma warning disable CS1591 public class PublisherSetupOptionsBuilder { private INamingPolicy? _namingPolicy; diff --git a/src/Blumchen/Serialization/IDictionaryExtensions.cs b/src/Blumchen/Serialization/IDictionaryExtensions.cs index f97da92..0017b82 100644 --- a/src/Blumchen/Serialization/IDictionaryExtensions.cs +++ b/src/Blumchen/Serialization/IDictionaryExtensions.cs @@ -1,4 +1,3 @@ -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member namespace Blumchen.Serialization; diff --git a/src/Blumchen/Serialization/INamingPolicy.cs b/src/Blumchen/Serialization/INamingPolicy.cs index 68c96ef..be1be55 100644 --- a/src/Blumchen/Serialization/INamingPolicy.cs +++ b/src/Blumchen/Serialization/INamingPolicy.cs @@ -1,5 +1,4 @@ namespace Blumchen.Serialization; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public interface INamingPolicy { diff --git a/src/Blumchen/Serialization/ITypeResolver.cs b/src/Blumchen/Serialization/ITypeResolver.cs index 049462b..71e49d3 100644 --- a/src/Blumchen/Serialization/ITypeResolver.cs +++ b/src/Blumchen/Serialization/ITypeResolver.cs @@ -3,7 +3,6 @@ using System.Text.Json.Serialization.Metadata; namespace Blumchen.Serialization; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public interface ITypeResolver { diff --git a/src/Blumchen/Serialization/JsonSerialization.cs b/src/Blumchen/Serialization/JsonSerialization.cs index feb4095..6fdfad0 100644 --- a/src/Blumchen/Serialization/JsonSerialization.cs +++ b/src/Blumchen/Serialization/JsonSerialization.cs @@ -4,7 +4,6 @@ using Blumchen.Streams; namespace Blumchen.Serialization; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public static class JsonSerialization { diff --git a/src/Blumchen/Serialization/MessageUrnAttribute.cs b/src/Blumchen/Serialization/MessageUrnAttribute.cs index 8963e34..11ae685 100644 --- a/src/Blumchen/Serialization/MessageUrnAttribute.cs +++ b/src/Blumchen/Serialization/MessageUrnAttribute.cs @@ -1,7 +1,6 @@ using System.Collections.Concurrent; namespace Blumchen.Serialization; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member [AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface)] public class MessageUrnAttribute: diff --git a/src/Blumchen/Subscriptions/IConsume.cs b/src/Blumchen/Subscriptions/IConsume.cs index 4a59c0f..6773455 100644 --- a/src/Blumchen/Subscriptions/IConsume.cs +++ b/src/Blumchen/Subscriptions/IConsume.cs @@ -1,5 +1,4 @@ namespace Blumchen.Subscriptions; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public interface IConsume; diff --git a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs index 64e6033..74750c1 100644 --- a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs +++ b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs @@ -3,7 +3,6 @@ using Npgsql; #pragma warning disable CA2208 -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member namespace Blumchen.Subscriptions.Management; diff --git a/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs b/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs index 4595dd4..dae42b6 100644 --- a/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs +++ b/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs @@ -5,7 +5,6 @@ namespace Blumchen.Subscriptions.Management; using static ReplicationSlotManagement.CreateReplicationSlotResult; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public static class ReplicationSlotManagement { diff --git a/src/Blumchen/Subscriptions/MimeType.cs b/src/Blumchen/Subscriptions/MimeType.cs index 8bb3ef9..1197908 100644 --- a/src/Blumchen/Subscriptions/MimeType.cs +++ b/src/Blumchen/Subscriptions/MimeType.cs @@ -1,6 +1,5 @@ namespace Blumchen.Subscriptions; -#pragma warning disable CS1591 public abstract record MimeType(string mimeType) { public record Json(): MimeType("application/json"); diff --git a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs index 90b75d2..1f74c3a 100644 --- a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs @@ -3,7 +3,6 @@ using Npgsql.Replication.PgOutput.Messages; namespace Blumchen.Subscriptions.Replication; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public interface IReplicationDataMapper { diff --git a/src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs b/src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs index 099a627..aa2e812 100644 --- a/src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs +++ b/src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs @@ -1,5 +1,4 @@ namespace Blumchen.Subscriptions.ReplicationMessageHandlers; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public interface IEnvelope; diff --git a/src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs b/src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs index 5603e79..eaaffe4 100644 --- a/src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs +++ b/src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs @@ -5,7 +5,6 @@ using Npgsql; namespace Blumchen.Subscriptions.SnapshotReader; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public static class SnapshotReader { diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 95854af..fa28b2e 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -12,7 +12,6 @@ using Npgsql.Replication.PgOutput.Messages; namespace Blumchen.Subscriptions; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member using static PublicationManagement; using static ReplicationSlotManagement; diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index 13a97d8..fb7e5be 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -5,7 +5,7 @@ using System.Text.Json.Serialization; namespace Blumchen.Subscriptions; -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public sealed class SubscriptionOptionsBuilder { private static string? _connectionString; From ad74d3c5a78b46438580c2e7533a0364fb307386 Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 4 Jul 2024 14:58:27 +0200 Subject: [PATCH 03/80] made UseTable() optional --- src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index fb7e5be..90cbdd0 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -90,9 +90,9 @@ public SubscriptionOptionsBuilder WithErrorProcessor(IErrorProcessor? errorProce internal ISubscriptionOptions Build() { + _messageTable ??= TableDescriptorBuilder.Build(); ArgumentNullException.ThrowIfNull(_connectionString); ArgumentNullException.ThrowIfNull(_jsonSerializerContext); - ArgumentNullException.ThrowIfNull(_messageTable); var typeResolver = new JsonTypeResolver(_jsonSerializerContext, _namingPolicy); foreach (var type in _registry.Keys) typeResolver.WhiteList(type); From b59d3869d1c4c6fa4c5938d128356d9732de815f Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 4 Jul 2024 15:11:02 +0200 Subject: [PATCH 04/80] enforce AOT --- src/Subscriber/Subscriber.csproj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Subscriber/Subscriber.csproj b/src/Subscriber/Subscriber.csproj index d5d5273..7878764 100644 --- a/src/Subscriber/Subscriber.csproj +++ b/src/Subscriber/Subscriber.csproj @@ -5,7 +5,8 @@ net8.0 enable enable - true + True + true false From 71c1f1419c20783bca8384d6dd573475535487e7 Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 4 Jul 2024 15:10:33 +0200 Subject: [PATCH 05/80] rename IConsumes to IHandles --- src/Blumchen/Subscriptions/IConsume.cs | 8 -------- src/Blumchen/Subscriptions/IHandler.cs | 8 ++++++++ src/Blumchen/Subscriptions/ISubscriptionOptions.cs | 4 ++-- src/Blumchen/Subscriptions/Subscription.cs | 12 ++++++------ .../Subscriptions/SubscriptionOptionsBuilder.cs | 10 +++++----- src/Subscriber/Program.cs | 8 ++++---- src/Tests/DatabaseFixture.cs | 8 ++++---- 7 files changed, 29 insertions(+), 29 deletions(-) delete mode 100644 src/Blumchen/Subscriptions/IConsume.cs create mode 100644 src/Blumchen/Subscriptions/IHandler.cs diff --git a/src/Blumchen/Subscriptions/IConsume.cs b/src/Blumchen/Subscriptions/IConsume.cs deleted file mode 100644 index 6773455..0000000 --- a/src/Blumchen/Subscriptions/IConsume.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Blumchen.Subscriptions; - -public interface IConsume; - -public interface IConsumes: IConsume where T : class -{ - Task Handle(T value); -} diff --git a/src/Blumchen/Subscriptions/IHandler.cs b/src/Blumchen/Subscriptions/IHandler.cs new file mode 100644 index 0000000..b722f25 --- /dev/null +++ b/src/Blumchen/Subscriptions/IHandler.cs @@ -0,0 +1,8 @@ +namespace Blumchen.Subscriptions; + +public interface IHandler; + +public interface IHandler: IHandler where T : class +{ + Task Handle(T value); +} diff --git a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs b/src/Blumchen/Subscriptions/ISubscriptionOptions.cs index 1be177c..79894e1 100644 --- a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs +++ b/src/Blumchen/Subscriptions/ISubscriptionOptions.cs @@ -19,7 +19,7 @@ void Deconstruct( out ReplicationSlotSetupOptions replicationSlotSetupOptions, out IErrorProcessor errorProcessor, out IReplicationDataMapper dataMapper, - out Dictionary registry); + out Dictionary registry); } internal record SubscriptionOptions( @@ -28,4 +28,4 @@ internal record SubscriptionOptions( ReplicationSlotSetupOptions ReplicationOptions, IErrorProcessor ErrorProcessor, IReplicationDataMapper DataMapper, - Dictionary Registry): ISubscriptionOptions; + Dictionary Registry): ISubscriptionOptions; diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index fa28b2e..3c6ee7c 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -88,7 +88,7 @@ public async IAsyncEnumerable Subscribe( private static async IAsyncEnumerable ProcessEnvelope( IEnvelope envelope, - Dictionary registry, + Dictionary registry, IErrorProcessor errorProcessor ) where T:class { @@ -109,14 +109,14 @@ IErrorProcessor errorProcessor } } - private static readonly Dictionary Cache = []; + private static readonly Dictionary Cache = []; - private static (IConsume consumer, MethodInfo methodInfo) Memoize + private static (IHandler consumer, MethodInfo methodInfo) Memoize ( - Dictionary registry, + Dictionary registry, Type objType, - Func, Type, (IConsume consumer, MethodInfo methodInfo)> func + Func, Type, (IHandler consumer, MethodInfo methodInfo)> func ) { if (!Cache.TryGetValue(objType, out var entry)) @@ -124,7 +124,7 @@ private static (IConsume consumer, MethodInfo methodInfo) Memoize Cache[objType] = entry; return entry; } - private static (IConsume consumer, MethodInfo methodInfo) Consumer(Dictionary registry, Type objType) + private static (IHandler consumer, MethodInfo methodInfo) Consumer(Dictionary registry, Type objType) { var consumer = registry[objType] ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); var methodInfos = consumer.GetType().GetMethods(BindingFlags.Instance|BindingFlags.Public); diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index 90cbdd0..9b7e7c5 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -12,7 +12,7 @@ public sealed class SubscriptionOptionsBuilder private static PublicationManagement.PublicationSetupOptions _publicationSetupOptions; private static ReplicationSlotManagement.ReplicationSlotSetupOptions? _replicationSlotSetupOptions; private static IReplicationDataMapper? _dataMapper; - private readonly Dictionary _registry = []; + private readonly Dictionary _registry = []; private IErrorProcessor? _errorProcessor; private INamingPolicy? _namingPolicy; private JsonSerializerContext? _jsonSerializerContext; @@ -74,10 +74,10 @@ public SubscriptionOptionsBuilder WithReplicationOptions(ReplicationSlotManageme } [UsedImplicitly] - public SubscriptionOptionsBuilder Consumes(TU consumer) where T : class - where TU : class, IConsumes + public SubscriptionOptionsBuilder Handles(TU handler) where T : class + where TU : class, IHandler { - _registry.TryAdd(typeof(T), consumer); + _registry.TryAdd(typeof(T), handler); return this; } @@ -119,7 +119,7 @@ static void Ensure(Func> evalFn, string formattedMsg) } } -public class ObjectTracingConsumer: IConsumes +public class ObjectTracingConsumer: IHandler { private static ulong _counter = 0; public Task Handle(object value) diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index c023ac0..ca08c10 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -30,8 +30,8 @@ ) .NamingPolicy(new AttributeNamingPolicy()) .JsonContext(SourceGenerationContext.Default) - .Consumes(consumer) - .Consumes(consumer), LoggerFactory.Create(builder => builder.AddConsole()), ct + .Handles(consumer) + .Handles(consumer), LoggerFactory.Create(builder => builder.AddConsole()), ct ).GetAsyncEnumerator(ct); await using var cursor1 = cursor.ConfigureAwait(false); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested); @@ -46,8 +46,8 @@ namespace Subscriber { internal class Consumer: - IConsumes, - IConsumes + IHandler, + IHandler { public Task Handle(UserCreatedContract value) => Console.Out.WriteLineAsync(JsonSerialization.ToJson(value, SourceGenerationContext.Default.UserCreatedContract)); public Task Handle(UserDeletedContract value) => Console.Out.WriteLineAsync(JsonSerialization.ToJson(value, SourceGenerationContext.Default.UserDeletedContract)); diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index fc71e14..9bc2e9a 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -17,7 +17,7 @@ public abstract class DatabaseFixture(ITestOutputHelper output): IAsyncLifetime { protected ITestOutputHelper Output { get; } = output; protected readonly Func TimeoutTokenSource = () => new(Debugger.IsAttached ? TimeSpan.FromHours(1) : TimeSpan.FromSeconds(2)); - protected class TestConsumer(Action log, JsonTypeInfo info): IConsumes where T : class + protected class TestHandler(Action log, JsonTypeInfo info): IHandler where T : class { public async Task Handle(T value) { @@ -67,7 +67,7 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri await command.ExecuteNonQueryAsync(ct); } - protected (TestConsumer consumer, SubscriptionOptionsBuilder subscriptionOptionsBuilder) SetupFor( + protected (TestHandler handler, SubscriptionOptionsBuilder subscriptionOptionsBuilder) SetupFor( string connectionString, string eventsTable, JsonSerializerContext info, @@ -78,13 +78,13 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri { var jsonTypeInfo = info.GetTypeInfo(typeof(T)); ArgumentNullException.ThrowIfNull(jsonTypeInfo); - var consumer = new TestConsumer(log, jsonTypeInfo); + var consumer = new TestHandler(log, jsonTypeInfo); var subscriptionOptionsBuilder = new SubscriptionOptionsBuilder() .WithErrorProcessor(new TestOutErrorProcessor(Output)) .ConnectionString(connectionString) .JsonContext(info) .NamingPolicy(namingPolicy) - .Consumes>(consumer) + .Handles>(consumer) .WithTable(o => o.Name(eventsTable)) .WithPublicationOptions( new PublicationManagement.PublicationSetupOptions(PublicationName: publicationName ?? Randomise("events_pub")) From 86656eced34a2beb19ee4a1eff52835f998609fc Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 4 Jul 2024 15:14:23 +0200 Subject: [PATCH 06/80] Use double quote --- .../Management/PublicationManagement.cs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs index 74750c1..c2b6f6c 100644 --- a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs +++ b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs @@ -69,14 +69,11 @@ internal static Task CreatePublication( ISet eventTypes, CancellationToken ct ) { + var sql = $"CREATE PUBLICATION \"{publicationName}\" FOR TABLE {tableName} {{0}} WITH (publish = 'insert');"; return eventTypes.Count switch { - 0 => Execute(dataSource, $"CREATE PUBLICATION {publicationName} FOR TABLE {tableName} WITH (publish = 'insert');", - ct - ), - _ => Execute(dataSource, $"CREATE PUBLICATION {publicationName} FOR TABLE {tableName} WHERE ({PublicationFilter(eventTypes)}) WITH (publish = 'insert');", - ct - ) + 0 => Execute(dataSource, string.Format(sql,string.Empty), ct), + _ => Execute(dataSource, string.Format(sql, $"WHERE ({PublicationFilter(eventTypes)})"), ct) }; static string PublicationFilter(ICollection input) => string.Join(" OR ", input.Select(s => $"message_type = '{s}'")); } @@ -128,8 +125,7 @@ private static Task PublicationExists( this NpgsqlDataSource dataSource, string publicationName, CancellationToken ct - ) => - dataSource.Exists("pg_publication", "pubname = $1", [publicationName], ct); + ) => dataSource.Exists("pg_publication", "pubname = $1", [publicationName], ct); public abstract record SetupPublicationResult { From 438cf21997e6021342b7cf580ea6f8151352ad0d Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 4 Jul 2024 15:15:13 +0200 Subject: [PATCH 07/80] use ILKE. Replication slot name is forced to lcase --- .../Management/ReplicationSlotManagement.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs b/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs index dae42b6..adeb6a8 100644 --- a/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs +++ b/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs @@ -9,6 +9,12 @@ namespace Blumchen.Subscriptions.Management; public static class ReplicationSlotManagement { #pragma warning disable CA2208 + private static Task ReplicationSlotExists( + this NpgsqlDataSource dataSource, + string slotName, + CancellationToken ct + ) => dataSource.Exists("pg_replication_slots", "slot_name ILIKE $1", [slotName], ct); + public static async Task SetupReplicationSlot( this NpgsqlDataSource dataSource, LogicalReplicationConnection connection, @@ -53,18 +59,12 @@ static async Task Create( } } - private static Task ReplicationSlotExists( - this NpgsqlDataSource dataSource, - string slotName, - CancellationToken ct - ) => dataSource.Exists("pg_replication_slots", "slot_name = $1", [slotName], ct); - public record ReplicationSlotSetupOptions( string SlotName = $"{TableDescriptorBuilder.MessageTable.DefaultName}_slot", Subscription.CreateStyle CreateStyle = Subscription.CreateStyle.WhenNotExists, - bool Binary = false //https://www.postgresql.org/docs/current/sql-createsubscription.html#SQL-CREATESUBSCRIPTION-WITH-BINARY + bool Binary = + false //https://www.postgresql.org/docs/current/sql-createsubscription.html#SQL-CREATESUBSCRIPTION-WITH-BINARY ); - public abstract record CreateReplicationSlotResult { public record None: CreateReplicationSlotResult; From b24c01f21feba7c1d41ad206992db6c914ea632b Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 4 Jul 2024 15:15:37 +0200 Subject: [PATCH 08/80] unused directive --- src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs index fd378d2..71e78e4 100644 --- a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs +++ b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs @@ -1,4 +1,3 @@ -using Blumchen; using Blumchen.Publications; using Blumchen.Serialization; using Blumchen.Subscriptions; From b024c6fe2161e1d532a418e92a2fe3bb5a14a686 Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 4 Jul 2024 16:36:11 +0200 Subject: [PATCH 09/80] remove static variables to enable multiple instances on same process --- src/Blumchen/Subscriptions/Subscription.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 3c6ee7c..57dd724 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -25,8 +25,8 @@ public enum CreateStyle AlwaysRecreate, Never } - private static LogicalReplicationConnection? _connection; - private static readonly SubscriptionOptionsBuilder Builder = new(); + private LogicalReplicationConnection? _connection; + private readonly SubscriptionOptionsBuilder _builder = new(); private ISubscriptionOptions? _options; public async IAsyncEnumerable Subscribe( Func builder, @@ -34,7 +34,7 @@ public async IAsyncEnumerable Subscribe( [EnumeratorCancellation] CancellationToken ct = default ) { - _options = builder(Builder).Build(); + _options = builder(_builder).Build(); var (connectionString, publicationSetupOptions, replicationSlotSetupOptions, errorProcessor, replicationDataMapper, registry) = _options; var dataSourceBuilder = new NpgsqlDataSourceBuilder(connectionString); dataSourceBuilder.UseLoggerFactory(loggerFactory); From 8cc1feddd39736e0d73d342f786fd2a5946942f5 Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 5 Jul 2024 00:50:15 +0200 Subject: [PATCH 10/80] Added DependencyIbjection project --- Blumchen.sln | 6 ++ .../Blumchen.DependencyInjection.csproj | 49 ++++++++++++++ .../Configuration/DatabaseOptions.cs | 2 + .../Workers/ServiceCollectionExtensions.cs | 13 ++++ .../Workers/Worker.cs | 64 +++++++++++++++++++ 5 files changed, 134 insertions(+) create mode 100644 src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj create mode 100644 src/Blumchen.DependencyInjection/Configuration/DatabaseOptions.cs create mode 100644 src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs create mode 100644 src/Blumchen.DependencyInjection/Workers/Worker.cs diff --git a/Blumchen.sln b/Blumchen.sln index a5020ed..6024139 100644 --- a/Blumchen.sln +++ b/Blumchen.sln @@ -43,6 +43,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "postgres", "postgres", "{8A EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "demo", "demo", "{A4044484-FE08-4399-8239-14AABFA30AD7}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Blumchen.DependencyInjection", "src\Blumchen.DependencyInjection\Blumchen.DependencyInjection.csproj", "{A07167E3-4CF7-40EF-8E55-A37A0F57B89D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -69,6 +71,10 @@ Global {2BBDA071-FB1C-4D62-A954-B22EA6B1C738}.Debug|Any CPU.Build.0 = Debug|Any CPU {2BBDA071-FB1C-4D62-A954-B22EA6B1C738}.Release|Any CPU.ActiveCfg = Release|Any CPU {2BBDA071-FB1C-4D62-A954-B22EA6B1C738}.Release|Any CPU.Build.0 = Release|Any CPU + {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj b/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj new file mode 100644 index 0000000..a808d2d --- /dev/null +++ b/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj @@ -0,0 +1,49 @@ + + + + 0.1.0 + net8.0 + true + true + true + false + true + true + true + true + 12.0 + Oskar Dudycz + + https://github.com/event-driven-io/Blumchen + MIT + https://github.com/event-driven-io/Blumchen.git + true + Blumchen + true + true + true + snupkg + Blumchen + true + enable + enable + + + + 1591 + + + + 1591 + + + + + + + + + + + + diff --git a/src/Blumchen.DependencyInjection/Configuration/DatabaseOptions.cs b/src/Blumchen.DependencyInjection/Configuration/DatabaseOptions.cs new file mode 100644 index 0000000..1be0098 --- /dev/null +++ b/src/Blumchen.DependencyInjection/Configuration/DatabaseOptions.cs @@ -0,0 +1,2 @@ +namespace Blumchen.Configuration; +public record DatabaseOptions(string ConnectionString); diff --git a/src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs b/src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..ec4e34c --- /dev/null +++ b/src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs @@ -0,0 +1,13 @@ +using Microsoft.Extensions.DependencyInjection; +#pragma warning disable IL2091 + +namespace Blumchen.Workers; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddBlumchen(this IServiceCollection service, T? instance = default) + where T : Worker where TU : class => + instance is null + ? service.AddHostedService() + : service.AddHostedService(_=>instance); +} diff --git a/src/Blumchen.DependencyInjection/Workers/Worker.cs b/src/Blumchen.DependencyInjection/Workers/Worker.cs new file mode 100644 index 0000000..1d06a31 --- /dev/null +++ b/src/Blumchen.DependencyInjection/Workers/Worker.cs @@ -0,0 +1,64 @@ +using System.Collections.Concurrent; +using System.Text.Json.Serialization; +using Blumchen.Configuration; +using Blumchen.Serialization; +using Blumchen.Subscriptions; +using Blumchen.Subscriptions.Management; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Polly; + + +namespace Blumchen.Workers; + +public abstract class Worker( + DatabaseOptions databaseOptions, + IHandler handler, + JsonSerializerContext jsonSerializerContext, + IErrorProcessor errorProcessor, + ResiliencePipeline pipeline, + INamingPolicy namingPolicy, + PublicationManagement.PublicationSetupOptions publicationSetupOptions, + ReplicationSlotManagement.ReplicationSlotSetupOptions replicationSlotSetupOptions, + ILoggerFactory loggerFactory): BackgroundService where T : class +{ + private readonly ILogger> _logger = loggerFactory.CreateLogger>(); + private string WorkerName { get; } = $"{nameof(Worker)}<{typeof(T).Name}>"; + private static readonly ConcurrentDictionary> _actions = new(StringComparer.OrdinalIgnoreCase); + private static void Notify(ILogger logger, LogLevel level, string template, params object[] parameters) + { + static Action LoggerAction(LogLevel ll, bool enabled) => + (ll, enabled) switch + { + (LogLevel.Information, true) => (logger, template, parameters) => logger.LogInformation(template, parameters), + (LogLevel.Debug, true) => (logger, template, parameters) => logger.LogDebug(template, parameters), + (_, _) => (_, __, ___) => { } + }; + _actions.GetOrAdd(template,s => LoggerAction(level, logger.IsEnabled(level)))(logger, template, parameters); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await pipeline.ExecuteAsync(async token => + { + await using var subscription = new Subscription(); + await using var cursor = subscription.Subscribe(builder => + builder + .ConnectionString(databaseOptions.ConnectionString) + .WithErrorProcessor(errorProcessor) + .Handles>(handler) + .NamingPolicy(namingPolicy) + .JsonContext(jsonSerializerContext) + .WithPublicationOptions(publicationSetupOptions) + .WithReplicationOptions(replicationSlotSetupOptions) + , ct: token, loggerFactory: loggerFactory).GetAsyncEnumerator(token); + Notify(_logger, LogLevel.Information,"{WorkerName} started", WorkerName); + while (await cursor.MoveNextAsync().ConfigureAwait(false) && !token.IsCancellationRequested) + Notify(_logger, LogLevel.Debug, "{cursor.Current} processed", cursor.Current); + + }, stoppingToken).ConfigureAwait(false); + Notify(_logger, LogLevel.Information, "{WorkerName} stopped", WorkerName); + return; + } + +} From 380d06aed86fc5b832295553e153b00394b196c1 Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 5 Jul 2024 00:52:04 +0200 Subject: [PATCH 11/80] Added demo projrct for DI --- Blumchen.sln | 7 +++ README.md | 2 +- src/SubscriberWorker/Contracts.cs | 22 ++++++++ src/SubscriberWorker/Handler.cs | 25 +++++++++ src/SubscriberWorker/Program.cs | 59 ++++++++++++++++++++ src/SubscriberWorker/SubscriberWorker.cs | 28 ++++++++++ src/SubscriberWorker/SubscriberWorker.csproj | 23 ++++++++ 7 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 src/SubscriberWorker/Contracts.cs create mode 100644 src/SubscriberWorker/Handler.cs create mode 100644 src/SubscriberWorker/Program.cs create mode 100644 src/SubscriberWorker/SubscriberWorker.cs create mode 100644 src/SubscriberWorker/SubscriberWorker.csproj diff --git a/Blumchen.sln b/Blumchen.sln index 6024139..fd99d48 100644 --- a/Blumchen.sln +++ b/Blumchen.sln @@ -45,6 +45,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "demo", "demo", "{A4044484-F EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Blumchen.DependencyInjection", "src\Blumchen.DependencyInjection\Blumchen.DependencyInjection.csproj", "{A07167E3-4CF7-40EF-8E55-A37A0F57B89D}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SubscriberWorker", "src\SubscriberWorker\SubscriberWorker.csproj", "{DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -75,6 +77,10 @@ Global {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Debug|Any CPU.Build.0 = Debug|Any CPU {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Release|Any CPU.ActiveCfg = Release|Any CPU {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Release|Any CPU.Build.0 = Release|Any CPU + {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -85,6 +91,7 @@ Global {F81E2D5B-FC59-4396-A911-56BE65E4FE80} = {A4044484-FE08-4399-8239-14AABFA30AD7} {C050E9E8-3FB6-4581-953F-31826E385FB4} = {CD59A1A0-F40D-4047-87A3-66C0F1519FA5} {8AAAA344-B5FD-48D9-B2BA-379E374448D4} = {CD59A1A0-F40D-4047-87A3-66C0F1519FA5} + {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92} = {A4044484-FE08-4399-8239-14AABFA30AD7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9A868C51-0460-4700-AF33-E1A921192614} diff --git a/README.md b/README.md index 48856b2..36fd616 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Main logic is placed in [EventsSubscription](./src/Blumchen/Subscriptions/Subscr ```shell docker-compose up ``` -2. Run(order doesn't matter) Publisher and Subscriber apps, under 'demo' folder, from vs-studio, and follow Publisher instructions. +2. Run(order doesn't matter) Publisher and (Subscriber or SubscriberWorker) apps, under 'demo' folder, from vs-studio, and follow Publisher instructions. ## Testing (against default docker instance) diff --git a/src/SubscriberWorker/Contracts.cs b/src/SubscriberWorker/Contracts.cs new file mode 100644 index 0000000..6fc3d0d --- /dev/null +++ b/src/SubscriberWorker/Contracts.cs @@ -0,0 +1,22 @@ +using System.Text.Json.Serialization; +using Blumchen.Serialization; + +namespace SubscriberWorker +{ + [MessageUrn("user-created:v1")] + public record UserCreatedContract( + Guid Id, + string Name + ); + + [MessageUrn("user-deleted:v1")] + public record UserDeletedContract( + Guid Id, + string Name + ); + + [JsonSourceGenerationOptions(WriteIndented = true)] + [JsonSerializable(typeof(UserCreatedContract))] + [JsonSerializable(typeof(UserDeletedContract))] + internal partial class SourceGenerationContext: JsonSerializerContext; +} diff --git a/src/SubscriberWorker/Handler.cs b/src/SubscriberWorker/Handler.cs new file mode 100644 index 0000000..95c003c --- /dev/null +++ b/src/SubscriberWorker/Handler.cs @@ -0,0 +1,25 @@ +using Blumchen.Subscriptions; +using Microsoft.Extensions.Logging; +#pragma warning disable CS9113 // Parameter is unread. + +namespace SubscriberWorker; + + +public class Handler(ILoggerFactory loggerFactory): IHandler where T : class +{ + private readonly ILogger _logger = loggerFactory.CreateLogger>(); + private Task ReportSuccess(int count) + { + if(_logger.IsEnabled(LogLevel.Debug)) + _logger.LogDebug($"Read #{count} messages {typeof(T).FullName}"); + return Task.CompletedTask; + } + + private int _counter; + private int _completed; + public Task Handle(T value) + => Interlocked.Increment(ref _counter) % 10 == 0 + //Simulating some exception on out of process dependencies + ? Task.FromException(new Exception($"Error on publishing {nameof(T)}")) + : ReportSuccess(Interlocked.Increment(ref _completed)); +} diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs new file mode 100644 index 0000000..6f224e4 --- /dev/null +++ b/src/SubscriberWorker/Program.cs @@ -0,0 +1,59 @@ +using System.Text.Json.Serialization; +using Blumchen.Configuration; +using Blumchen.Serialization; +using Blumchen.Subscriptions; +using Blumchen.Workers; +using Commons; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Polly.Retry; +using Polly; +using SubscriberWorker; + + +#pragma warning disable CS8601 // Possible null reference assignment. +Console.Title = typeof(Program).Assembly.GetName().Name; +#pragma warning restore CS8601 // Possible null reference assignment. + + + +AppDomain.CurrentDomain.UnhandledException += (_, e) => Console.Out.WriteLine(e.ExceptionObject.ToString()); +TaskScheduler.UnobservedTaskException += (_, e) => Console.Out.WriteLine(e.Exception.ToString()); + +var cancellationTokenSource = new CancellationTokenSource(); +var builder = Host.CreateApplicationBuilder(args); + +builder.Services + .AddBlumchen, UserCreatedContract>() + .AddSingleton, Handler>() + .AddBlumchen, UserDeletedContract>() + .AddSingleton, Handler>() + + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton(new DatabaseOptions(Settings.ConnectionString)) + .AddResiliencePipeline("default",(pipelineBuilder,context) => + pipelineBuilder + .AddRetry(new RetryStrategyOptions + { + BackoffType = DelayBackoffType.Constant, + Delay = TimeSpan.FromSeconds(5), + MaxRetryAttempts = int.MaxValue + }).Build()) + .AddLogging(loggingBuilder => + { + loggingBuilder + .AddFilter("Microsoft", LogLevel.Warning) + .AddFilter("System", LogLevel.Warning) + .AddFilter("Npgsql", LogLevel.Information) + .AddFilter("Blumchen", LogLevel.Debug) + .AddFilter("SubscriberWorker", LogLevel.Debug) + .AddSimpleConsole(); + }); + +await builder + .Build() + .RunAsync(cancellationTokenSource.Token) + .ConfigureAwait(false); diff --git a/src/SubscriberWorker/SubscriberWorker.cs b/src/SubscriberWorker/SubscriberWorker.cs new file mode 100644 index 0000000..5b97440 --- /dev/null +++ b/src/SubscriberWorker/SubscriberWorker.cs @@ -0,0 +1,28 @@ +using System.Text.Json.Serialization; +using Blumchen.Configuration; +using Blumchen.Serialization; +using Blumchen.Subscriptions; +using Blumchen.Subscriptions.Management; +using Blumchen.Workers; +using Microsoft.Extensions.Logging; +using Polly.Registry; +// ReSharper disable ClassNeverInstantiated.Global + +namespace SubscriberWorker; +public class SubscriberWorker( + DatabaseOptions databaseOptions, + IHandler handler, + JsonSerializerContext jsonSerializerContext, + ResiliencePipelineProvider pipelineProvider, + INamingPolicy namingPolicy, + IErrorProcessor errorProcessor, + ILoggerFactory loggerFactory +): Worker(databaseOptions + , handler + , jsonSerializerContext + , errorProcessor + , pipelineProvider.GetPipeline("default") + , namingPolicy + , new PublicationManagement.PublicationSetupOptions($"{typeof(T).Name}_pub") + , new ReplicationSlotManagement.ReplicationSlotSetupOptions($"{typeof(T).Name}_slot") + , loggerFactory) where T : class; diff --git a/src/SubscriberWorker/SubscriberWorker.csproj b/src/SubscriberWorker/SubscriberWorker.csproj new file mode 100644 index 0000000..cfe99a0 --- /dev/null +++ b/src/SubscriberWorker/SubscriberWorker.csproj @@ -0,0 +1,23 @@ + + + + Exe + net8.0 + enable + enable + true + true + false + + + + + + + + + + + + + From acf5f4dcd1ed1410f2a2324fe9f805820fec66cb Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 5 Jul 2024 10:22:49 +0200 Subject: [PATCH 12/80] Enhanced logging --- src/Publisher/Program.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Publisher/Program.cs b/src/Publisher/Program.cs index dae2987..adc4a9a 100644 --- a/src/Publisher/Program.cs +++ b/src/Publisher/Program.cs @@ -22,7 +22,7 @@ if (line != null && int.TryParse(line, out var result)) { var cts = new CancellationTokenSource(); - + var messages = result / 3; var ct = cts.Token; var connection = new NpgsqlConnection(Settings.ConnectionString); await using var connection1 = connection.ConfigureAwait(false); @@ -36,6 +36,9 @@ 1 => new UserDeleted(Guid.NewGuid()), _ => new UserModified(Guid.NewGuid()) }); + await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 0) ? 1 : 0)} {nameof(UserCreated)}"); + await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 1) ? 1 : 0)} {nameof(UserDeleted)}"); + await Console.Out.WriteLineAsync($"Publishing {messages} {nameof(UserModified)}"); foreach (var @event in @events) { var transaction = await connection.BeginTransactionAsync(ct).ConfigureAwait(false); @@ -63,6 +66,7 @@ throw; } } + await Console.Out.WriteLineAsync($"Published {result} messages!"); } //use a batch command //{ From e918dfaca1bc6f94b28134e5b3b566370f176c4e Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 5 Jul 2024 16:10:32 +0200 Subject: [PATCH 13/80] bumped version to 0.1.1 --- .../Blumchen.DependencyInjection.csproj | 2 +- src/Blumchen/Blumchen.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj b/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj index a808d2d..5b9e0e5 100644 --- a/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj +++ b/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj @@ -1,7 +1,7 @@ - 0.1.0 + 0.1.1 net8.0 true true diff --git a/src/Blumchen/Blumchen.csproj b/src/Blumchen/Blumchen.csproj index 0f26919..72bdb54 100644 --- a/src/Blumchen/Blumchen.csproj +++ b/src/Blumchen/Blumchen.csproj @@ -1,7 +1,7 @@ - 0.1.0 + 0.1.1 net8.0 true true From 2055fede86e05e100261830ee849076b529c4a23 Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 5 Jul 2024 18:52:59 +0200 Subject: [PATCH 14/80] allow tableDescriptor access --- src/Blumchen.DependencyInjection/Workers/Worker.cs | 2 ++ src/Blumchen/MessageTableOptions.cs | 2 ++ src/SubscriberWorker/SubscriberWorker.cs | 1 + 3 files changed, 5 insertions(+) diff --git a/src/Blumchen.DependencyInjection/Workers/Worker.cs b/src/Blumchen.DependencyInjection/Workers/Worker.cs index 1d06a31..ba45bb7 100644 --- a/src/Blumchen.DependencyInjection/Workers/Worker.cs +++ b/src/Blumchen.DependencyInjection/Workers/Worker.cs @@ -20,6 +20,7 @@ public abstract class Worker( INamingPolicy namingPolicy, PublicationManagement.PublicationSetupOptions publicationSetupOptions, ReplicationSlotManagement.ReplicationSlotSetupOptions replicationSlotSetupOptions, + Func tableDescriptorBuilder, ILoggerFactory loggerFactory): BackgroundService where T : class { private readonly ILogger> _logger = loggerFactory.CreateLogger>(); @@ -45,6 +46,7 @@ await pipeline.ExecuteAsync(async token => await using var cursor = subscription.Subscribe(builder => builder .ConnectionString(databaseOptions.ConnectionString) + .WithTable(tableDescriptorBuilder) .WithErrorProcessor(errorProcessor) .Handles>(handler) .NamingPolicy(namingPolicy) diff --git a/src/Blumchen/MessageTableOptions.cs b/src/Blumchen/MessageTableOptions.cs index 7216108..b038de3 100644 --- a/src/Blumchen/MessageTableOptions.cs +++ b/src/Blumchen/MessageTableOptions.cs @@ -33,6 +33,8 @@ public TableDescriptorBuilder MessageType(string name, int dimension = 250) return this; } + public TableDescriptorBuilder UseDefaults() => this; + public record MessageTable(string Name = MessageTable.DefaultName) { internal const string DefaultName = "outbox"; diff --git a/src/SubscriberWorker/SubscriberWorker.cs b/src/SubscriberWorker/SubscriberWorker.cs index 5b97440..d6c4a67 100644 --- a/src/SubscriberWorker/SubscriberWorker.cs +++ b/src/SubscriberWorker/SubscriberWorker.cs @@ -25,4 +25,5 @@ ILoggerFactory loggerFactory , namingPolicy , new PublicationManagement.PublicationSetupOptions($"{typeof(T).Name}_pub") , new ReplicationSlotManagement.ReplicationSlotSetupOptions($"{typeof(T).Name}_slot") + , tableDescriptorBuilder => tableDescriptorBuilder.UseDefaults() , loggerFactory) where T : class; From eadba73a6878176e00bec7340e1719790b7398f8 Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 5 Jul 2024 18:58:09 +0200 Subject: [PATCH 15/80] renamed vars --- src/Blumchen.DependencyInjection/Workers/Worker.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Blumchen.DependencyInjection/Workers/Worker.cs b/src/Blumchen.DependencyInjection/Workers/Worker.cs index ba45bb7..8e147fc 100644 --- a/src/Blumchen.DependencyInjection/Workers/Worker.cs +++ b/src/Blumchen.DependencyInjection/Workers/Worker.cs @@ -25,7 +25,7 @@ public abstract class Worker( { private readonly ILogger> _logger = loggerFactory.CreateLogger>(); private string WorkerName { get; } = $"{nameof(Worker)}<{typeof(T).Name}>"; - private static readonly ConcurrentDictionary> _actions = new(StringComparer.OrdinalIgnoreCase); + private static readonly ConcurrentDictionary> LoggingActions = new(StringComparer.OrdinalIgnoreCase); private static void Notify(ILogger logger, LogLevel level, string template, params object[] parameters) { static Action LoggerAction(LogLevel ll, bool enabled) => @@ -35,7 +35,7 @@ static Action LoggerAction(LogLevel ll, bool enabled) (LogLevel.Debug, true) => (logger, template, parameters) => logger.LogDebug(template, parameters), (_, _) => (_, __, ___) => { } }; - _actions.GetOrAdd(template,s => LoggerAction(level, logger.IsEnabled(level)))(logger, template, parameters); + LoggingActions.GetOrAdd(template,s => LoggerAction(level, logger.IsEnabled(level)))(logger, template, parameters); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) From c097047d399b2af0b7c0fe8d5964cdaaa5f392ec Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 15 Jul 2024 13:54:45 +0200 Subject: [PATCH 16/80] expose NpgsqlDataSource along with connection string - close #16 --- .../Configuration/DatabaseOptions.cs | 2 -- .../Workers/Worker.cs | 23 +++++++++--------- src/Blumchen/Serialization/ITypeResolver.cs | 4 ++-- .../Subscriptions/ISubscriptionOptions.cs | 10 +++++--- src/Blumchen/Subscriptions/Subscription.cs | 11 +++------ .../SubscriptionOptionsBuilder.cs | 21 ++++++++++++---- src/Subscriber/Program.cs | 6 ++++- src/SubscriberWorker/Program.cs | 24 ++++++++++--------- src/SubscriberWorker/SubscriberWorker.cs | 12 ++++++---- src/Tests/DatabaseFixture.cs | 1 + src/Tests/When_Subscription_Already_Exists.cs | 2 +- ...ption_Does_Not_Exist_And_Table_Is_Empty.cs | 2 +- ...n_Does_Not_Exist_And_Table_Is_Not_Empty.cs | 2 +- 13 files changed, 69 insertions(+), 51 deletions(-) delete mode 100644 src/Blumchen.DependencyInjection/Configuration/DatabaseOptions.cs diff --git a/src/Blumchen.DependencyInjection/Configuration/DatabaseOptions.cs b/src/Blumchen.DependencyInjection/Configuration/DatabaseOptions.cs deleted file mode 100644 index 1be0098..0000000 --- a/src/Blumchen.DependencyInjection/Configuration/DatabaseOptions.cs +++ /dev/null @@ -1,2 +0,0 @@ -namespace Blumchen.Configuration; -public record DatabaseOptions(string ConnectionString); diff --git a/src/Blumchen.DependencyInjection/Workers/Worker.cs b/src/Blumchen.DependencyInjection/Workers/Worker.cs index 8e147fc..6a07118 100644 --- a/src/Blumchen.DependencyInjection/Workers/Worker.cs +++ b/src/Blumchen.DependencyInjection/Workers/Worker.cs @@ -1,18 +1,19 @@ using System.Collections.Concurrent; using System.Text.Json.Serialization; -using Blumchen.Configuration; using Blumchen.Serialization; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Management; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Npgsql; using Polly; namespace Blumchen.Workers; public abstract class Worker( - DatabaseOptions databaseOptions, + NpgsqlDataSource dataSource, + string connectionString, IHandler handler, JsonSerializerContext jsonSerializerContext, IErrorProcessor errorProcessor, @@ -21,9 +22,8 @@ public abstract class Worker( PublicationManagement.PublicationSetupOptions publicationSetupOptions, ReplicationSlotManagement.ReplicationSlotSetupOptions replicationSlotSetupOptions, Func tableDescriptorBuilder, - ILoggerFactory loggerFactory): BackgroundService where T : class + ILogger logger): BackgroundService where T : class { - private readonly ILogger> _logger = loggerFactory.CreateLogger>(); private string WorkerName { get; } = $"{nameof(Worker)}<{typeof(T).Name}>"; private static readonly ConcurrentDictionary> LoggingActions = new(StringComparer.OrdinalIgnoreCase); private static void Notify(ILogger logger, LogLevel level, string template, params object[] parameters) @@ -33,9 +33,9 @@ static Action LoggerAction(LogLevel ll, bool enabled) { (LogLevel.Information, true) => (logger, template, parameters) => logger.LogInformation(template, parameters), (LogLevel.Debug, true) => (logger, template, parameters) => logger.LogDebug(template, parameters), - (_, _) => (_, __, ___) => { } + (_, _) => (_, _, _) => { } }; - LoggingActions.GetOrAdd(template,s => LoggerAction(level, logger.IsEnabled(level)))(logger, template, parameters); + LoggingActions.GetOrAdd(template,_ => LoggerAction(level, logger.IsEnabled(level)))(logger, template, parameters); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -45,7 +45,8 @@ await pipeline.ExecuteAsync(async token => await using var subscription = new Subscription(); await using var cursor = subscription.Subscribe(builder => builder - .ConnectionString(databaseOptions.ConnectionString) + .DataSource(dataSource) + .ConnectionString(connectionString) .WithTable(tableDescriptorBuilder) .WithErrorProcessor(errorProcessor) .Handles>(handler) @@ -53,13 +54,13 @@ await pipeline.ExecuteAsync(async token => .JsonContext(jsonSerializerContext) .WithPublicationOptions(publicationSetupOptions) .WithReplicationOptions(replicationSlotSetupOptions) - , ct: token, loggerFactory: loggerFactory).GetAsyncEnumerator(token); - Notify(_logger, LogLevel.Information,"{WorkerName} started", WorkerName); + , ct: token).GetAsyncEnumerator(token); + Notify(logger, LogLevel.Information,"{WorkerName} started", WorkerName); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !token.IsCancellationRequested) - Notify(_logger, LogLevel.Debug, "{cursor.Current} processed", cursor.Current); + Notify(logger, LogLevel.Debug, "{cursor.Current} processed", cursor.Current); }, stoppingToken).ConfigureAwait(false); - Notify(_logger, LogLevel.Information, "{WorkerName} stopped", WorkerName); + Notify(logger, LogLevel.Information, "{WorkerName} stopped", WorkerName); return; } diff --git a/src/Blumchen/Serialization/ITypeResolver.cs b/src/Blumchen/Serialization/ITypeResolver.cs index 71e49d3..cdddbf4 100644 --- a/src/Blumchen/Serialization/ITypeResolver.cs +++ b/src/Blumchen/Serialization/ITypeResolver.cs @@ -24,8 +24,8 @@ internal sealed class JsonTypeResolver( internal void WhiteList(Type type) { var typeInfo = SerializationContext.GetTypeInfo(type) ?? throw new NotSupportedException(type.FullName); - _typeDictionary.AddOrUpdate(_namingPolicy.Bind(typeInfo.Type), _ => typeInfo.Type, (s,t) =>typeInfo.Type); - _typeInfoDictionary.AddOrUpdate(typeInfo.Type, _ => typeInfo, (_,__)=> typeInfo); + _typeDictionary.AddOrUpdate(_namingPolicy.Bind(typeInfo.Type), _ => typeInfo.Type, (_,_) =>typeInfo.Type); + _typeInfoDictionary.AddOrUpdate(typeInfo.Type, _ => typeInfo, (_,_)=> typeInfo); } public (string, JsonTypeInfo) Resolve(Type type) => diff --git a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs b/src/Blumchen/Subscriptions/ISubscriptionOptions.cs index 79894e1..83a383c 100644 --- a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs +++ b/src/Blumchen/Subscriptions/ISubscriptionOptions.cs @@ -1,5 +1,6 @@ using Blumchen.Subscriptions.Replication; using JetBrains.Annotations; +using Npgsql; using static Blumchen.Subscriptions.Management.PublicationManagement; using static Blumchen.Subscriptions.Management.ReplicationSlotManagement; @@ -7,14 +8,16 @@ namespace Blumchen.Subscriptions; internal interface ISubscriptionOptions { - [UsedImplicitly] string ConnectionString { get; } + [UsedImplicitly] NpgsqlDataSource DataSource { get; } + [UsedImplicitly] NpgsqlConnectionStringBuilder ConnectionStringBuilder { get; } IReplicationDataMapper DataMapper { get; } [UsedImplicitly] PublicationSetupOptions PublicationOptions { get; } [UsedImplicitly] ReplicationSlotSetupOptions ReplicationOptions { get; } [UsedImplicitly] IErrorProcessor ErrorProcessor { get; } void Deconstruct( - out string connectionString, + out NpgsqlDataSource dataSource, + out NpgsqlConnectionStringBuilder connectionStringBuilder, out PublicationSetupOptions publicationSetupOptions, out ReplicationSlotSetupOptions replicationSlotSetupOptions, out IErrorProcessor errorProcessor, @@ -23,7 +26,8 @@ void Deconstruct( } internal record SubscriptionOptions( - string ConnectionString, + NpgsqlDataSource DataSource, + NpgsqlConnectionStringBuilder ConnectionStringBuilder, PublicationSetupOptions PublicationOptions, ReplicationSlotSetupOptions ReplicationOptions, IErrorProcessor ErrorProcessor, diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 57dd724..a06adcb 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -30,22 +30,17 @@ public enum CreateStyle private ISubscriptionOptions? _options; public async IAsyncEnumerable Subscribe( Func builder, - ILoggerFactory? loggerFactory = null, [EnumeratorCancellation] CancellationToken ct = default ) { _options = builder(_builder).Build(); - var (connectionString, publicationSetupOptions, replicationSlotSetupOptions, errorProcessor, replicationDataMapper, registry) = _options; - var dataSourceBuilder = new NpgsqlDataSourceBuilder(connectionString); - dataSourceBuilder.UseLoggerFactory(loggerFactory); - - var dataSource = dataSourceBuilder.Build(); + var (dataSource, connectionStringBuilder, publicationSetupOptions, replicationSlotSetupOptions, errorProcessor, replicationDataMapper, registry) = _options; + await dataSource.EnsureTableExists(publicationSetupOptions.TableDescriptor, ct).ConfigureAwait(false); - _connection = new LogicalReplicationConnection(connectionString); + _connection = new LogicalReplicationConnection(connectionStringBuilder.ConnectionString); await _connection.Open(ct).ConfigureAwait(false); - await dataSource.SetupPublication(publicationSetupOptions, ct).ConfigureAwait(false); var result = await dataSource.SetupReplicationSlot(_connection, replicationSlotSetupOptions, ct).ConfigureAwait(false); diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index 9b7e7c5..8c4365b 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -2,13 +2,15 @@ using Blumchen.Subscriptions.Management; using Blumchen.Subscriptions.Replication; using JetBrains.Annotations; +using Npgsql; using System.Text.Json.Serialization; namespace Blumchen.Subscriptions; public sealed class SubscriptionOptionsBuilder { - private static string? _connectionString; + private static NpgsqlConnectionStringBuilder? _connectionStringBuilder; + private static NpgsqlDataSource? _dataSource; private static PublicationManagement.PublicationSetupOptions _publicationSetupOptions; private static ReplicationSlotManagement.ReplicationSlotSetupOptions? _replicationSlotSetupOptions; private static IReplicationDataMapper? _dataMapper; @@ -22,7 +24,7 @@ public sealed class SubscriptionOptionsBuilder static SubscriptionOptionsBuilder() { - _connectionString = null; + _connectionStringBuilder = default; _publicationSetupOptions = new(); _replicationSlotSetupOptions = default; _dataMapper = default; @@ -40,7 +42,14 @@ public SubscriptionOptionsBuilder WithTable( [UsedImplicitly] public SubscriptionOptionsBuilder ConnectionString(string connectionString) { - _connectionString = connectionString; + _connectionStringBuilder = new NpgsqlConnectionStringBuilder(connectionString); + return this; + } + + [UsedImplicitly] + public SubscriptionOptionsBuilder DataSource(NpgsqlDataSource dataSource) + { + _dataSource = dataSource; return this; } @@ -91,7 +100,8 @@ public SubscriptionOptionsBuilder WithErrorProcessor(IErrorProcessor? errorProce internal ISubscriptionOptions Build() { _messageTable ??= TableDescriptorBuilder.Build(); - ArgumentNullException.ThrowIfNull(_connectionString); + ArgumentNullException.ThrowIfNull(_connectionStringBuilder); + ArgumentNullException.ThrowIfNull(_dataSource); ArgumentNullException.ThrowIfNull(_jsonSerializerContext); var typeResolver = new JsonTypeResolver(_jsonSerializerContext, _namingPolicy); @@ -104,7 +114,8 @@ internal ISubscriptionOptions Build() if (_registry.Count == 0)_registry.Add(typeof(object), new ObjectTracingConsumer()); return new SubscriptionOptions( - _connectionString, + _dataSource, + _connectionStringBuilder, _publicationSetupOptions, _replicationSlotSetupOptions ?? new ReplicationSlotManagement.ReplicationSlotSetupOptions(), _errorProcessor ?? new ConsoleOutErrorProcessor(), diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index ca08c10..eabee73 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -2,6 +2,7 @@ using Blumchen.Subscriptions; using Commons; using Microsoft.Extensions.Logging; +using Npgsql; using Subscriber; #pragma warning disable CS8601 // Possible null reference assignment. @@ -20,8 +21,11 @@ try { + var dataSourceBuilder = new NpgsqlDataSourceBuilder(Settings.ConnectionString) + .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())); var cursor = subscription.Subscribe( builder => builder + .DataSource(dataSourceBuilder.Build()) .ConnectionString(Settings.ConnectionString) .WithTable(options => options .Id("id") @@ -31,7 +35,7 @@ .NamingPolicy(new AttributeNamingPolicy()) .JsonContext(SourceGenerationContext.Default) .Handles(consumer) - .Handles(consumer), LoggerFactory.Create(builder => builder.AddConsole()), ct + .Handles(consumer), ct:ct ).GetAsyncEnumerator(ct); await using var cursor1 = cursor.ConfigureAwait(false); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested); diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index 6f224e4..3349b40 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -1,5 +1,4 @@ using System.Text.Json.Serialization; -using Blumchen.Configuration; using Blumchen.Serialization; using Blumchen.Subscriptions; using Blumchen.Workers; @@ -10,6 +9,7 @@ using Polly.Retry; using Polly; using SubscriberWorker; +using Npgsql; #pragma warning disable CS8601 // Possible null reference assignment. @@ -29,19 +29,21 @@ .AddSingleton, Handler>() .AddBlumchen, UserDeletedContract>() .AddSingleton, Handler>() - + .AddSingleton(Settings.ConnectionString) + .AddTransient(sp => + new NpgsqlDataSourceBuilder(Settings.ConnectionString) + .UseLoggerFactory(sp.GetRequiredService()).Build()) .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(new DatabaseOptions(Settings.ConnectionString)) - .AddResiliencePipeline("default",(pipelineBuilder,context) => + .AddResiliencePipeline("default", (pipelineBuilder, _) => pipelineBuilder - .AddRetry(new RetryStrategyOptions - { - BackoffType = DelayBackoffType.Constant, - Delay = TimeSpan.FromSeconds(5), - MaxRetryAttempts = int.MaxValue - }).Build()) + .AddRetry(new RetryStrategyOptions + { + BackoffType = DelayBackoffType.Constant, + Delay = TimeSpan.FromSeconds(5), + MaxRetryAttempts = int.MaxValue + }).Build()) .AddLogging(loggingBuilder => { loggingBuilder @@ -51,7 +53,7 @@ .AddFilter("Blumchen", LogLevel.Debug) .AddFilter("SubscriberWorker", LogLevel.Debug) .AddSimpleConsole(); - }); + }).AddSingleton(sp => sp.GetRequiredService().CreateLogger()); await builder .Build() diff --git a/src/SubscriberWorker/SubscriberWorker.cs b/src/SubscriberWorker/SubscriberWorker.cs index d6c4a67..cb28195 100644 --- a/src/SubscriberWorker/SubscriberWorker.cs +++ b/src/SubscriberWorker/SubscriberWorker.cs @@ -1,23 +1,25 @@ using System.Text.Json.Serialization; -using Blumchen.Configuration; using Blumchen.Serialization; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Management; using Blumchen.Workers; using Microsoft.Extensions.Logging; +using Npgsql; using Polly.Registry; // ReSharper disable ClassNeverInstantiated.Global namespace SubscriberWorker; public class SubscriberWorker( - DatabaseOptions databaseOptions, + NpgsqlDataSource dataSource, + string connectionString, IHandler handler, JsonSerializerContext jsonSerializerContext, ResiliencePipelineProvider pipelineProvider, INamingPolicy namingPolicy, IErrorProcessor errorProcessor, - ILoggerFactory loggerFactory -): Worker(databaseOptions + ILogger logger +): Worker(dataSource + , connectionString , handler , jsonSerializerContext , errorProcessor @@ -26,4 +28,4 @@ ILoggerFactory loggerFactory , new PublicationManagement.PublicationSetupOptions($"{typeof(T).Name}_pub") , new ReplicationSlotManagement.ReplicationSlotSetupOptions($"{typeof(T).Name}_slot") , tableDescriptorBuilder => tableDescriptorBuilder.UseDefaults() - , loggerFactory) where T : class; + , logger) where T : class; diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index 9bc2e9a..19d0af5 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -81,6 +81,7 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri var consumer = new TestHandler(log, jsonTypeInfo); var subscriptionOptionsBuilder = new SubscriptionOptionsBuilder() .WithErrorProcessor(new TestOutErrorProcessor(Output)) + .DataSource(new NpgsqlDataSourceBuilder(connectionString).Build()) .ConnectionString(connectionString) .JsonContext(info) .NamingPolicy(namingPolicy) diff --git a/src/Tests/When_Subscription_Already_Exists.cs b/src/Tests/When_Subscription_Already_Exists.cs index 4575b69..71b78c9 100644 --- a/src/Tests/When_Subscription_Already_Exists.cs +++ b/src/Tests/When_Subscription_Already_Exists.cs @@ -46,7 +46,7 @@ await MessageAppender.AppendAsync( var subscription = new Subscription(); await using var subscription1 = subscription.ConfigureAwait(false); - await foreach (var envelope in subscription.Subscribe(_ => subscriptionOptions, null, ct).ConfigureAwait(false)) + await foreach (var envelope in subscription.Subscribe(_ => subscriptionOptions, ct).ConfigureAwait(false)) { Assert.Equal(@expected, ((OkEnvelope)envelope).Value); return; diff --git a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs index 71e78e4..3a52e23 100644 --- a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs +++ b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs @@ -38,7 +38,7 @@ public async Task Read_from_table_using_named_transaction_snapshot() SubscriberContext.Default, sharedNamingPolicy, Output.WriteLine); var subscription = new Subscription(); await using var subscription1 = subscription.ConfigureAwait(false); - await foreach (var envelope in subscription.Subscribe(_ => subscriptionOptions, null, ct).ConfigureAwait(false)) + await foreach (var envelope in subscription.Subscribe(_ => subscriptionOptions, ct).ConfigureAwait(false)) { Assert.Equal(@expected, ((OkEnvelope)envelope).Value); return; diff --git a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs index 6422396..d1029fb 100644 --- a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs +++ b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs @@ -40,7 +40,7 @@ public async Task Read_from_table_using_named_transaction_snapshot() var subscription = new Subscription(); await using var subscription1 = subscription.ConfigureAwait(false); - await foreach (var envelope in subscription.Subscribe(_ => subscriptionOptions, null, ct).ConfigureAwait(false)) + await foreach (var envelope in subscription.Subscribe(_ => subscriptionOptions, ct).ConfigureAwait(false)) { Assert.Equal(@expected, ((OkEnvelope)envelope).Value); return; From ef9ac954e253c316e97cc32bd7e8a6d6325b1740 Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 15 Jul 2024 14:56:40 +0200 Subject: [PATCH 17/80] rename extension method --- src/Blumchen.DependencyInjection/Workers/Worker.cs | 2 +- src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs | 2 +- src/Subscriber/Program.cs | 4 ++-- src/Tests/DatabaseFixture.cs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Blumchen.DependencyInjection/Workers/Worker.cs b/src/Blumchen.DependencyInjection/Workers/Worker.cs index 6a07118..2d69ccd 100644 --- a/src/Blumchen.DependencyInjection/Workers/Worker.cs +++ b/src/Blumchen.DependencyInjection/Workers/Worker.cs @@ -49,7 +49,7 @@ await pipeline.ExecuteAsync(async token => .ConnectionString(connectionString) .WithTable(tableDescriptorBuilder) .WithErrorProcessor(errorProcessor) - .Handles>(handler) + .Consumes>(handler) .NamingPolicy(namingPolicy) .JsonContext(jsonSerializerContext) .WithPublicationOptions(publicationSetupOptions) diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index 8c4365b..cf39401 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -83,7 +83,7 @@ public SubscriptionOptionsBuilder WithReplicationOptions(ReplicationSlotManageme } [UsedImplicitly] - public SubscriptionOptionsBuilder Handles(TU handler) where T : class + public SubscriptionOptionsBuilder Consumes(TU handler) where T : class where TU : class, IHandler { _registry.TryAdd(typeof(T), handler); diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index eabee73..8c8e057 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -34,8 +34,8 @@ ) .NamingPolicy(new AttributeNamingPolicy()) .JsonContext(SourceGenerationContext.Default) - .Handles(consumer) - .Handles(consumer), ct:ct + .Consumes(consumer) + .Consumes(consumer), ct:ct ).GetAsyncEnumerator(ct); await using var cursor1 = cursor.ConfigureAwait(false); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested); diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index 19d0af5..78e1525 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -85,7 +85,7 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri .ConnectionString(connectionString) .JsonContext(info) .NamingPolicy(namingPolicy) - .Handles>(consumer) + .Consumes>(consumer) .WithTable(o => o.Name(eventsTable)) .WithPublicationOptions( new PublicationManagement.PublicationSetupOptions(PublicationName: publicationName ?? Randomise("events_pub")) From 345cad6f1b0f67075b5eeb9523ea86c8f1b45b51 Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 15 Jul 2024 15:01:19 +0200 Subject: [PATCH 18/80] move to files --- src/Blumchen/Subscriptions/IErrorProcessor.cs | 11 ++++++++++ .../Subscriptions/ObjectTracingConsumer.cs | 11 ++++++++++ .../SubscriptionOptionsBuilder.cs | 20 ------------------- src/SubscriberWorker/Handler.cs | 7 +++---- 4 files changed, 25 insertions(+), 24 deletions(-) create mode 100644 src/Blumchen/Subscriptions/IErrorProcessor.cs create mode 100644 src/Blumchen/Subscriptions/ObjectTracingConsumer.cs diff --git a/src/Blumchen/Subscriptions/IErrorProcessor.cs b/src/Blumchen/Subscriptions/IErrorProcessor.cs new file mode 100644 index 0000000..b3cec9d --- /dev/null +++ b/src/Blumchen/Subscriptions/IErrorProcessor.cs @@ -0,0 +1,11 @@ +namespace Blumchen.Subscriptions; + +public interface IErrorProcessor +{ + Func Process { get; } +} + +public record ConsoleOutErrorProcessor: IErrorProcessor +{ + public Func Process => exception => Console.Out.WriteLineAsync($"record id:{0} resulted in error:{exception.Message}"); +} diff --git a/src/Blumchen/Subscriptions/ObjectTracingConsumer.cs b/src/Blumchen/Subscriptions/ObjectTracingConsumer.cs new file mode 100644 index 0000000..3263482 --- /dev/null +++ b/src/Blumchen/Subscriptions/ObjectTracingConsumer.cs @@ -0,0 +1,11 @@ +namespace Blumchen.Subscriptions; + +internal class ObjectTracingConsumer: IHandler +{ + private static ulong _counter = 0; + public Task Handle(object value) + { + Interlocked.Increment(ref _counter); + return Console.Out.WriteLineAsync(); + } +} diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index cf39401..22162a1 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -129,23 +129,3 @@ static void Ensure(Func> evalFn, string formattedMsg) } } - -public class ObjectTracingConsumer: IHandler -{ - private static ulong _counter = 0; - public Task Handle(object value) - { - Interlocked.Increment(ref _counter); - return Console.Out.WriteLineAsync(); - } -} - -public interface IErrorProcessor -{ - Func Process { get; } -} - -public record ConsoleOutErrorProcessor: IErrorProcessor -{ - public Func Process => exception => Console.Out.WriteLineAsync($"record id:{0} resulted in error:{exception.Message}"); -} diff --git a/src/SubscriberWorker/Handler.cs b/src/SubscriberWorker/Handler.cs index 95c003c..070fc8c 100644 --- a/src/SubscriberWorker/Handler.cs +++ b/src/SubscriberWorker/Handler.cs @@ -5,13 +5,12 @@ namespace SubscriberWorker; -public class Handler(ILoggerFactory loggerFactory): IHandler where T : class +public class Handler(ILogger logger): IHandler where T : class { - private readonly ILogger _logger = loggerFactory.CreateLogger>(); private Task ReportSuccess(int count) { - if(_logger.IsEnabled(LogLevel.Debug)) - _logger.LogDebug($"Read #{count} messages {typeof(T).FullName}"); + if(logger.IsEnabled(LogLevel.Debug)) + logger.LogDebug($"Read #{count} messages {typeof(T).FullName}"); return Task.CompletedTask; } From 645ec1a0bb7c4f487068250c033d3a169a704a79 Mon Sep 17 00:00:00 2001 From: giordanol Date: Tue, 16 Jul 2024 19:13:29 +0200 Subject: [PATCH 19/80] simplified di registration --- .../Blumchen.DependencyInjection.csproj | 5 ++ .../Workers/ServiceCollectionExtensions.cs | 21 +++++-- .../Workers/Worker.cs | 38 ++----------- .../Workers/WorkerOptionsBuilder.cs | 37 +++++++++++++ src/Blumchen/Blumchen.csproj | 3 + .../Subscriptions/ISubscriptionOptions.cs | 2 +- src/Blumchen/Subscriptions/Subscription.cs | 20 +++++-- .../SubscriptionOptionsBuilder.cs | 31 ++++------- src/Publisher/Contracts.cs | 8 ++- src/Publisher/Program.cs | 14 +++-- src/Subscriber/Program.cs | 4 +- src/SubscriberWorker/Contracts.cs | 7 +++ src/SubscriberWorker/Handler.cs | 29 +++++++--- src/SubscriberWorker/Program.cs | 55 ++++++++++++++++--- src/SubscriberWorker/SubscriberWorker.cs | 31 ----------- src/Tests/DatabaseFixture.cs | 2 +- 16 files changed, 186 insertions(+), 121 deletions(-) create mode 100644 src/Blumchen.DependencyInjection/Workers/WorkerOptionsBuilder.cs delete mode 100644 src/SubscriberWorker/SubscriberWorker.cs diff --git a/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj b/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj index 5b9e0e5..bb407ba 100644 --- a/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj +++ b/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj @@ -38,6 +38,11 @@ + + all + none + all + diff --git a/src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs b/src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs index ec4e34c..0c8a6a0 100644 --- a/src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs +++ b/src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs @@ -1,13 +1,24 @@ +using Blumchen.Subscriptions; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Polly; + #pragma warning disable IL2091 namespace Blumchen.Workers; public static class ServiceCollectionExtensions { - public static IServiceCollection AddBlumchen(this IServiceCollection service, T? instance = default) - where T : Worker where TU : class => - instance is null - ? service.AddHostedService() - : service.AddHostedService(_=>instance); + + public static IServiceCollection AddBlumchen( + this IServiceCollection service, + Func workerOptions) + where T : class, IHandler => + service + .AddKeyedSingleton(typeof(T), (provider, _) => workerOptions(provider, new WorkerOptionsBuilder()).Build()) + .AddHostedService(provider => + new Worker(workerOptions(provider, new WorkerOptionsBuilder()).Build(), + provider.GetRequiredService>>())); + + } diff --git a/src/Blumchen.DependencyInjection/Workers/Worker.cs b/src/Blumchen.DependencyInjection/Workers/Worker.cs index 2d69ccd..dbca462 100644 --- a/src/Blumchen.DependencyInjection/Workers/Worker.cs +++ b/src/Blumchen.DependencyInjection/Workers/Worker.cs @@ -1,28 +1,13 @@ using System.Collections.Concurrent; -using System.Text.Json.Serialization; -using Blumchen.Serialization; using Blumchen.Subscriptions; -using Blumchen.Subscriptions.Management; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using Npgsql; -using Polly; - namespace Blumchen.Workers; -public abstract class Worker( - NpgsqlDataSource dataSource, - string connectionString, - IHandler handler, - JsonSerializerContext jsonSerializerContext, - IErrorProcessor errorProcessor, - ResiliencePipeline pipeline, - INamingPolicy namingPolicy, - PublicationManagement.PublicationSetupOptions publicationSetupOptions, - ReplicationSlotManagement.ReplicationSlotSetupOptions replicationSlotSetupOptions, - Func tableDescriptorBuilder, - ILogger logger): BackgroundService where T : class +public class Worker( + WorkerOptions options, + ILogger> logger): BackgroundService where T : class, IHandler { private string WorkerName { get; } = $"{nameof(Worker)}<{typeof(T).Name}>"; private static readonly ConcurrentDictionary> LoggingActions = new(StringComparer.OrdinalIgnoreCase); @@ -40,28 +25,17 @@ static Action LoggerAction(LogLevel ll, bool enabled) protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await pipeline.ExecuteAsync(async token => + await options.ResiliencePipeline.ExecuteAsync(async token => { await using var subscription = new Subscription(); - await using var cursor = subscription.Subscribe(builder => - builder - .DataSource(dataSource) - .ConnectionString(connectionString) - .WithTable(tableDescriptorBuilder) - .WithErrorProcessor(errorProcessor) - .Consumes>(handler) - .NamingPolicy(namingPolicy) - .JsonContext(jsonSerializerContext) - .WithPublicationOptions(publicationSetupOptions) - .WithReplicationOptions(replicationSlotSetupOptions) - , ct: token).GetAsyncEnumerator(token); + await using var cursor = subscription.Subscribe(options.SubscriptionOptions, ct: token) + .GetAsyncEnumerator(token); Notify(logger, LogLevel.Information,"{WorkerName} started", WorkerName); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !token.IsCancellationRequested) Notify(logger, LogLevel.Debug, "{cursor.Current} processed", cursor.Current); }, stoppingToken).ConfigureAwait(false); Notify(logger, LogLevel.Information, "{WorkerName} stopped", WorkerName); - return; } } diff --git a/src/Blumchen.DependencyInjection/Workers/WorkerOptionsBuilder.cs b/src/Blumchen.DependencyInjection/Workers/WorkerOptionsBuilder.cs new file mode 100644 index 0000000..f146b75 --- /dev/null +++ b/src/Blumchen.DependencyInjection/Workers/WorkerOptionsBuilder.cs @@ -0,0 +1,37 @@ +using Blumchen.Subscriptions; +using Polly; + +namespace Blumchen.Workers; + +public record WorkerOptions(ResiliencePipeline ResiliencePipeline, ISubscriptionOptions SubscriptionOptions); + +public interface IWorkerOptionsBuilder +{ + IWorkerOptionsBuilder ResiliencyPipeline(ResiliencePipeline resiliencePipeline); + IWorkerOptionsBuilder Subscription(Func? builder); + WorkerOptions Build(); +} + +internal sealed class WorkerOptionsBuilder: IWorkerOptionsBuilder +{ + private ResiliencePipeline? _resiliencePipeline = default; + private Func? _builder; + + public IWorkerOptionsBuilder ResiliencyPipeline(ResiliencePipeline resiliencePipeline) + { + _resiliencePipeline = resiliencePipeline; + return this; + }public IWorkerOptionsBuilder Subscription(Func? builder) + { + _builder = builder; + return this; + } + + public WorkerOptions Build() + { + ArgumentNullException.ThrowIfNull(_resiliencePipeline); + ArgumentNullException.ThrowIfNull(_builder); + return new(_resiliencePipeline, _builder(new SubscriptionOptionsBuilder()).Build()); + } +} + diff --git a/src/Blumchen/Blumchen.csproj b/src/Blumchen/Blumchen.csproj index 72bdb54..9847f66 100644 --- a/src/Blumchen/Blumchen.csproj +++ b/src/Blumchen/Blumchen.csproj @@ -39,6 +39,9 @@ <_Parameter1>Tests + + <_Parameter1>Blumchen.DependencyInjection + diff --git a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs b/src/Blumchen/Subscriptions/ISubscriptionOptions.cs index 83a383c..27dd513 100644 --- a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs +++ b/src/Blumchen/Subscriptions/ISubscriptionOptions.cs @@ -6,7 +6,7 @@ namespace Blumchen.Subscriptions; -internal interface ISubscriptionOptions +public interface ISubscriptionOptions { [UsedImplicitly] NpgsqlDataSource DataSource { get; } [UsedImplicitly] NpgsqlConnectionStringBuilder ConnectionStringBuilder { get; } diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index a06adcb..872df0b 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -5,7 +5,6 @@ using Blumchen.Subscriptions.Management; using Blumchen.Subscriptions.ReplicationMessageHandlers; using Blumchen.Subscriptions.SnapshotReader; -using Microsoft.Extensions.Logging; using Npgsql; using Npgsql.Replication; using Npgsql.Replication.PgOutput; @@ -33,9 +32,18 @@ public async IAsyncEnumerable Subscribe( [EnumeratorCancellation] CancellationToken ct = default ) { - _options = builder(_builder).Build(); - var (dataSource, connectionStringBuilder, publicationSetupOptions, replicationSlotSetupOptions, errorProcessor, replicationDataMapper, registry) = _options; - + await foreach (var _ in Subscribe(builder(_builder).Build(), ct)) + yield return _; + } + + internal async IAsyncEnumerable Subscribe( + ISubscriptionOptions subscriptionOptions, + [EnumeratorCancellation] CancellationToken ct = default + ) + { + _options = subscriptionOptions; + var (dataSource, connectionStringBuilder, publicationSetupOptions, replicationSlotSetupOptions, errorProcessor, replicationDataMapper, registry) = subscriptionOptions; + await dataSource.EnsureTableExists(publicationSetupOptions.TableDescriptor, ct).ConfigureAwait(false); _connection = new LogicalReplicationConnection(connectionStringBuilder.ConnectionString); @@ -60,8 +68,8 @@ public async IAsyncEnumerable Subscribe( ); await foreach (var envelope in ReadExistingRowsFromSnapshot(dataSource, created.SnapshotName, _options, ct).ConfigureAwait(false)) - await foreach (var subscribe in ProcessEnvelope(envelope, registry, errorProcessor).WithCancellation(ct).ConfigureAwait(false)) - yield return subscribe; + await foreach (var subscribe in ProcessEnvelope(envelope, registry, errorProcessor).WithCancellation(ct).ConfigureAwait(false)) + yield return subscribe; } await foreach (var message in diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index 22162a1..8166ab2 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -9,33 +9,23 @@ namespace Blumchen.Subscriptions; public sealed class SubscriptionOptionsBuilder { - private static NpgsqlConnectionStringBuilder? _connectionStringBuilder; - private static NpgsqlDataSource? _dataSource; - private static PublicationManagement.PublicationSetupOptions _publicationSetupOptions; - private static ReplicationSlotManagement.ReplicationSlotSetupOptions? _replicationSlotSetupOptions; - private static IReplicationDataMapper? _dataMapper; + private NpgsqlConnectionStringBuilder? _connectionStringBuilder; + private NpgsqlDataSource? _dataSource; + private PublicationManagement.PublicationSetupOptions _publicationSetupOptions = new(); + private ReplicationSlotManagement.ReplicationSlotSetupOptions? _replicationSlotSetupOptions; + private IReplicationDataMapper? _dataMapper; private readonly Dictionary _registry = []; private IErrorProcessor? _errorProcessor; private INamingPolicy? _namingPolicy; private JsonSerializerContext? _jsonSerializerContext; - private static readonly TableDescriptorBuilder TableDescriptorBuilder = new(); + private readonly TableDescriptorBuilder _tableDescriptorBuilder = new(); private TableDescriptorBuilder.MessageTable? _messageTable; - - - static SubscriptionOptionsBuilder() - { - _connectionStringBuilder = default; - _publicationSetupOptions = new(); - _replicationSlotSetupOptions = default; - _dataMapper = default; - } - [UsedImplicitly] public SubscriptionOptionsBuilder WithTable( Func builder) { - _messageTable = builder(TableDescriptorBuilder).Build(); + _messageTable = builder(_tableDescriptorBuilder).Build(); return this; } @@ -83,8 +73,7 @@ public SubscriptionOptionsBuilder WithReplicationOptions(ReplicationSlotManageme } [UsedImplicitly] - public SubscriptionOptionsBuilder Consumes(TU handler) where T : class - where TU : class, IHandler + public SubscriptionOptionsBuilder Consumes(IHandler handler) where T : class { _registry.TryAdd(typeof(T), handler); return this; @@ -99,7 +88,7 @@ public SubscriptionOptionsBuilder WithErrorProcessor(IErrorProcessor? errorProce internal ISubscriptionOptions Build() { - _messageTable ??= TableDescriptorBuilder.Build(); + _messageTable ??= _tableDescriptorBuilder.Build(); ArgumentNullException.ThrowIfNull(_connectionStringBuilder); ArgumentNullException.ThrowIfNull(_dataSource); ArgumentNullException.ThrowIfNull(_jsonSerializerContext); @@ -121,11 +110,11 @@ internal ISubscriptionOptions Build() _errorProcessor ?? new ConsoleOutErrorProcessor(), _dataMapper, _registry); + static void Ensure(Func> evalFn, string formattedMsg) { var misses = evalFn().ToArray(); if (misses.Length > 0) throw new Exception(string.Format(formattedMsg, string.Join(", ", misses.Select(t => $"'{t.Name}'")))); } - } } diff --git a/src/Publisher/Contracts.cs b/src/Publisher/Contracts.cs index e9995e9..8a10b46 100644 --- a/src/Publisher/Contracts.cs +++ b/src/Publisher/Contracts.cs @@ -16,15 +16,21 @@ internal record UserDeleted( string Name = "Deleted" ): IContract; -[MessageUrn("user-modified:v1")] //subscription ignored +[MessageUrn("user-modified:v1")] internal record UserModified( Guid Id, string Name = "Modified" ): IContract; +[MessageUrn("user-subscribed:v1")] +internal record UserSubscribed( + Guid Id, + string Name = "Subscribed" +): IContract; [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(UserCreated))] [JsonSerializable(typeof(UserDeleted))] [JsonSerializable(typeof(UserModified))] +[JsonSerializable(typeof(UserSubscribed))] internal partial class SourceGenerationContext: JsonSerializerContext; diff --git a/src/Publisher/Program.cs b/src/Publisher/Program.cs index adc4a9a..41da1a0 100644 --- a/src/Publisher/Program.cs +++ b/src/Publisher/Program.cs @@ -6,6 +6,7 @@ using UserCreated = Publisher.UserCreated; using UserDeleted = Publisher.UserDeleted; using UserModified = Publisher.UserModified; +using UserSubscribed = Publisher.UserSubscribed; Console.Title = typeof(Program).Assembly.GetName().Name!; Console.WriteLine("How many messages do you want to publish?(press CTRL+C to exit):"); @@ -22,7 +23,7 @@ if (line != null && int.TryParse(line, out var result)) { var cts = new CancellationTokenSource(); - var messages = result / 3; + var messages = result / 4; var ct = cts.Token; var connection = new NpgsqlConnection(Settings.ConnectionString); await using var connection1 = connection.ConfigureAwait(false); @@ -30,15 +31,17 @@ //use a command for each message { var @events = Enumerable.Range(0, result).Select(i => - (i % 3) switch + (i % 4) switch { 0 => new UserCreated(Guid.NewGuid()) as object, 1 => new UserDeleted(Guid.NewGuid()), - _ => new UserModified(Guid.NewGuid()) + 2 => new UserModified(Guid.NewGuid()), + _ => new UserSubscribed(Guid.NewGuid()) }); await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 0) ? 1 : 0)} {nameof(UserCreated)}"); await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 1) ? 1 : 0)} {nameof(UserDeleted)}"); - await Console.Out.WriteLineAsync($"Publishing {messages} {nameof(UserModified)}"); + await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 2) ? 1 : 0)} {nameof(UserModified)}"); + await Console.Out.WriteLineAsync($"Publishing {messages} {nameof(UserSubscribed)}"); foreach (var @event in @events) { var transaction = await connection.BeginTransactionAsync(ct).ConfigureAwait(false); @@ -55,6 +58,9 @@ case UserModified m: await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); break; + case UserSubscribed m: + await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); + break; } await transaction.CommitAsync(ct).ConfigureAwait(false); diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index 8c8e057..1d129f8 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -34,8 +34,8 @@ ) .NamingPolicy(new AttributeNamingPolicy()) .JsonContext(SourceGenerationContext.Default) - .Consumes(consumer) - .Consumes(consumer), ct:ct + .Consumes(consumer) + .Consumes(consumer), ct:ct ).GetAsyncEnumerator(ct); await using var cursor1 = cursor.ConfigureAwait(false); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested); diff --git a/src/SubscriberWorker/Contracts.cs b/src/SubscriberWorker/Contracts.cs index 6fc3d0d..a3c501a 100644 --- a/src/SubscriberWorker/Contracts.cs +++ b/src/SubscriberWorker/Contracts.cs @@ -15,8 +15,15 @@ public record UserDeletedContract( string Name ); + [MessageUrn("user-modified:v1")] //subscription ignored + public record UserModifiedContract( + Guid Id, + string Name = "Modified" + ); + [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(UserCreatedContract))] [JsonSerializable(typeof(UserDeletedContract))] + [JsonSerializable(typeof(UserModifiedContract))] internal partial class SourceGenerationContext: JsonSerializerContext; } diff --git a/src/SubscriberWorker/Handler.cs b/src/SubscriberWorker/Handler.cs index 070fc8c..3ecf3fb 100644 --- a/src/SubscriberWorker/Handler.cs +++ b/src/SubscriberWorker/Handler.cs @@ -4,21 +4,34 @@ namespace SubscriberWorker; - -public class Handler(ILogger logger): IHandler where T : class +public class HandlerBase(ILogger logger) { - private Task ReportSuccess(int count) + private Task ReportSuccess(int count) { - if(logger.IsEnabled(LogLevel.Debug)) + if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug($"Read #{count} messages {typeof(T).FullName}"); return Task.CompletedTask; } private int _counter; private int _completed; - public Task Handle(T value) - => Interlocked.Increment(ref _counter) % 10 == 0 + protected Task Handle(T value) => + Interlocked.Increment(ref _counter) % 10 == 0 //Simulating some exception on out of process dependencies - ? Task.FromException(new Exception($"Error on publishing {nameof(T)}")) - : ReportSuccess(Interlocked.Increment(ref _completed)); + ? Task.FromException(new Exception($"Error on publishing {typeof(T).FullName}")) + : ReportSuccess(Interlocked.Increment(ref _completed)); +} + +public class HandleImpl2(ILogger logger): HandlerBase(logger), + IHandler +{ + public Task Handle(UserDeletedContract value) => Handle(value); +} + +public class HandleImpl1(ILogger logger) : HandlerBase(logger), + IHandler, IHandler +{ + public Task Handle(UserCreatedContract value) => Handle(value); + + public Task Handle(UserModifiedContract value) => Handle(value); } diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index 3349b40..33a5188 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -10,6 +10,8 @@ using Polly; using SubscriberWorker; using Npgsql; +using Blumchen.Subscriptions.Management; +using Polly.Registry; #pragma warning disable CS8601 // Possible null reference assignment. @@ -25,17 +27,16 @@ var builder = Host.CreateApplicationBuilder(args); builder.Services - .AddBlumchen, UserCreatedContract>() - .AddSingleton, Handler>() - .AddBlumchen, UserDeletedContract>() - .AddSingleton, Handler>() - .AddSingleton(Settings.ConnectionString) - .AddTransient(sp => - new NpgsqlDataSourceBuilder(Settings.ConnectionString) - .UseLoggerFactory(sp.GetRequiredService()).Build()) + + .AddSingleton, HandleImpl1>() + .AddSingleton, HandleImpl1>() + .AddSingleton, HandleImpl2>() + .AddSingleton() + .AddSingleton() .AddSingleton() + .AddResiliencePipeline("default", (pipelineBuilder, _) => pipelineBuilder .AddRetry(new RetryStrategyOptions @@ -44,6 +45,7 @@ Delay = TimeSpan.FromSeconds(5), MaxRetryAttempts = int.MaxValue }).Build()) + .AddLogging(loggingBuilder => { loggingBuilder @@ -53,7 +55,42 @@ .AddFilter("Blumchen", LogLevel.Debug) .AddFilter("SubscriberWorker", LogLevel.Debug) .AddSimpleConsole(); - }).AddSingleton(sp => sp.GetRequiredService().CreateLogger()); + }) + .AddTransient(sp => + new NpgsqlDataSourceBuilder(Settings.ConnectionString) + .UseLoggerFactory(sp.GetRequiredService()).Build()) + + .AddBlumchen((provider, workerOptions) => + workerOptions + .Subscription(subscriptionOptions => + subscriptionOptions + .ConnectionString(Settings.ConnectionString) + .DataSource(provider.GetRequiredService()) + .WithReplicationOptions(new ReplicationSlotManagement.ReplicationSlotSetupOptions($"{nameof(HandleImpl1)}_slot")) + .WithPublicationOptions(new PublicationManagement.PublicationSetupOptions($"{nameof(HandleImpl1)}_pub")) + .WithErrorProcessor(provider.GetRequiredService()) + .NamingPolicy(provider.GetRequiredService()) + .JsonContext(SourceGenerationContext.Default) + .Consumes(provider.GetRequiredService>()) + .Consumes(provider.GetRequiredService>())) + .ResiliencyPipeline(provider.GetRequiredService>().GetPipeline("default")) + + ) + + .AddBlumchen((provider, workerOptions) => + workerOptions + .Subscription(subscriptionOptions => + subscriptionOptions.ConnectionString(Settings.ConnectionString) + .DataSource(provider.GetRequiredService()) + .WithReplicationOptions(new ReplicationSlotManagement.ReplicationSlotSetupOptions($"{nameof(HandleImpl2)}_slot")) + .WithPublicationOptions(new PublicationManagement.PublicationSetupOptions($"{nameof(HandleImpl2)}_pub")) + .WithErrorProcessor(provider.GetRequiredService()) + .NamingPolicy(provider.GetRequiredService()) + .JsonContext(SourceGenerationContext.Default) + .Consumes(provider.GetRequiredService>())) + .ResiliencyPipeline(provider.GetRequiredService>().GetPipeline("default")) + ) + ; await builder .Build() diff --git a/src/SubscriberWorker/SubscriberWorker.cs b/src/SubscriberWorker/SubscriberWorker.cs deleted file mode 100644 index cb28195..0000000 --- a/src/SubscriberWorker/SubscriberWorker.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Text.Json.Serialization; -using Blumchen.Serialization; -using Blumchen.Subscriptions; -using Blumchen.Subscriptions.Management; -using Blumchen.Workers; -using Microsoft.Extensions.Logging; -using Npgsql; -using Polly.Registry; -// ReSharper disable ClassNeverInstantiated.Global - -namespace SubscriberWorker; -public class SubscriberWorker( - NpgsqlDataSource dataSource, - string connectionString, - IHandler handler, - JsonSerializerContext jsonSerializerContext, - ResiliencePipelineProvider pipelineProvider, - INamingPolicy namingPolicy, - IErrorProcessor errorProcessor, - ILogger logger -): Worker(dataSource - , connectionString - , handler - , jsonSerializerContext - , errorProcessor - , pipelineProvider.GetPipeline("default") - , namingPolicy - , new PublicationManagement.PublicationSetupOptions($"{typeof(T).Name}_pub") - , new ReplicationSlotManagement.ReplicationSlotSetupOptions($"{typeof(T).Name}_slot") - , tableDescriptorBuilder => tableDescriptorBuilder.UseDefaults() - , logger) where T : class; diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index 78e1525..1b2f1af 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -85,7 +85,7 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri .ConnectionString(connectionString) .JsonContext(info) .NamingPolicy(namingPolicy) - .Consumes>(consumer) + .Consumes(consumer) .WithTable(o => o.Name(eventsTable)) .WithPublicationOptions( new PublicationManagement.PublicationSetupOptions(PublicationName: publicationName ?? Randomise("events_pub")) From ca28054932dd18a6b3abd30e308aad82994edb9f Mon Sep 17 00:00:00 2001 From: giordanol Date: Tue, 16 Jul 2024 19:28:05 +0200 Subject: [PATCH 20/80] collapse project --- Blumchen.sln | 6 ------ .../Blumchen.DependencyInjection.csproj | 2 +- src/Blumchen/Blumchen.csproj | 4 +++- .../Workers/ServiceCollectionExtensions.cs | 3 +-- .../Workers/Worker.cs | 0 .../Workers/WorkerOptionsBuilder.cs | 0 src/SubscriberWorker/Program.cs | 2 +- src/SubscriberWorker/SubscriberWorker.csproj | 1 - 8 files changed, 6 insertions(+), 12 deletions(-) rename src/{Blumchen.DependencyInjection => Blumchen}/Workers/ServiceCollectionExtensions.cs (87%) rename src/{Blumchen.DependencyInjection => Blumchen}/Workers/Worker.cs (100%) rename src/{Blumchen.DependencyInjection => Blumchen}/Workers/WorkerOptionsBuilder.cs (100%) diff --git a/Blumchen.sln b/Blumchen.sln index fd99d48..f8510ef 100644 --- a/Blumchen.sln +++ b/Blumchen.sln @@ -43,8 +43,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "postgres", "postgres", "{8A EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "demo", "demo", "{A4044484-FE08-4399-8239-14AABFA30AD7}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Blumchen.DependencyInjection", "src\Blumchen.DependencyInjection\Blumchen.DependencyInjection.csproj", "{A07167E3-4CF7-40EF-8E55-A37A0F57B89D}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SubscriberWorker", "src\SubscriberWorker\SubscriberWorker.csproj", "{DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}" EndProject Global @@ -73,10 +71,6 @@ Global {2BBDA071-FB1C-4D62-A954-B22EA6B1C738}.Debug|Any CPU.Build.0 = Debug|Any CPU {2BBDA071-FB1C-4D62-A954-B22EA6B1C738}.Release|Any CPU.ActiveCfg = Release|Any CPU {2BBDA071-FB1C-4D62-A954-B22EA6B1C738}.Release|Any CPU.Build.0 = Release|Any CPU - {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A07167E3-4CF7-40EF-8E55-A37A0F57B89D}.Release|Any CPU.Build.0 = Release|Any CPU {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}.Debug|Any CPU.Build.0 = Debug|Any CPU {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj b/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj index bb407ba..5c7b408 100644 --- a/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj +++ b/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj @@ -38,7 +38,7 @@ - + all none all diff --git a/src/Blumchen/Blumchen.csproj b/src/Blumchen/Blumchen.csproj index 9847f66..0138080 100644 --- a/src/Blumchen/Blumchen.csproj +++ b/src/Blumchen/Blumchen.csproj @@ -45,13 +45,15 @@ - + all none all + + diff --git a/src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs b/src/Blumchen/Workers/ServiceCollectionExtensions.cs similarity index 87% rename from src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs rename to src/Blumchen/Workers/ServiceCollectionExtensions.cs index 0c8a6a0..6950b04 100644 --- a/src/Blumchen.DependencyInjection/Workers/ServiceCollectionExtensions.cs +++ b/src/Blumchen/Workers/ServiceCollectionExtensions.cs @@ -1,7 +1,6 @@ using Blumchen.Subscriptions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using Polly; #pragma warning disable IL2091 @@ -18,7 +17,7 @@ public static IServiceCollection AddBlumchen( .AddKeyedSingleton(typeof(T), (provider, _) => workerOptions(provider, new WorkerOptionsBuilder()).Build()) .AddHostedService(provider => new Worker(workerOptions(provider, new WorkerOptionsBuilder()).Build(), - provider.GetRequiredService>>())); + ServiceProviderServiceExtensions.GetRequiredService>>(provider))); } diff --git a/src/Blumchen.DependencyInjection/Workers/Worker.cs b/src/Blumchen/Workers/Worker.cs similarity index 100% rename from src/Blumchen.DependencyInjection/Workers/Worker.cs rename to src/Blumchen/Workers/Worker.cs diff --git a/src/Blumchen.DependencyInjection/Workers/WorkerOptionsBuilder.cs b/src/Blumchen/Workers/WorkerOptionsBuilder.cs similarity index 100% rename from src/Blumchen.DependencyInjection/Workers/WorkerOptionsBuilder.cs rename to src/Blumchen/Workers/WorkerOptionsBuilder.cs diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index 33a5188..735b15d 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -1,7 +1,6 @@ using System.Text.Json.Serialization; using Blumchen.Serialization; using Blumchen.Subscriptions; -using Blumchen.Workers; using Commons; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -11,6 +10,7 @@ using SubscriberWorker; using Npgsql; using Blumchen.Subscriptions.Management; +using Blumchen.Workers; using Polly.Registry; diff --git a/src/SubscriberWorker/SubscriberWorker.csproj b/src/SubscriberWorker/SubscriberWorker.csproj index cfe99a0..ad5c3f6 100644 --- a/src/SubscriberWorker/SubscriberWorker.csproj +++ b/src/SubscriberWorker/SubscriberWorker.csproj @@ -16,7 +16,6 @@ - From f4cfe0cf8a844345696b802f31ee93d6df48a980 Mon Sep 17 00:00:00 2001 From: giordanol Date: Tue, 16 Jul 2024 19:28:25 +0200 Subject: [PATCH 21/80] rename folder --- .../ServiceCollectionExtensions.cs | 2 +- src/Blumchen/{Workers => DependencyInjection}/Worker.cs | 2 +- .../{Workers => DependencyInjection}/WorkerOptionsBuilder.cs | 2 +- src/SubscriberWorker/Program.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename src/Blumchen/{Workers => DependencyInjection}/ServiceCollectionExtensions.cs (95%) rename src/Blumchen/{Workers => DependencyInjection}/Worker.cs (98%) rename src/Blumchen/{Workers => DependencyInjection}/WorkerOptionsBuilder.cs (96%) diff --git a/src/Blumchen/Workers/ServiceCollectionExtensions.cs b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs similarity index 95% rename from src/Blumchen/Workers/ServiceCollectionExtensions.cs rename to src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs index 6950b04..c01d5ab 100644 --- a/src/Blumchen/Workers/ServiceCollectionExtensions.cs +++ b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs @@ -4,7 +4,7 @@ #pragma warning disable IL2091 -namespace Blumchen.Workers; +namespace Blumchen.DependencyInjection; public static class ServiceCollectionExtensions { diff --git a/src/Blumchen/Workers/Worker.cs b/src/Blumchen/DependencyInjection/Worker.cs similarity index 98% rename from src/Blumchen/Workers/Worker.cs rename to src/Blumchen/DependencyInjection/Worker.cs index dbca462..4f137b7 100644 --- a/src/Blumchen/Workers/Worker.cs +++ b/src/Blumchen/DependencyInjection/Worker.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -namespace Blumchen.Workers; +namespace Blumchen.DependencyInjection; public class Worker( WorkerOptions options, diff --git a/src/Blumchen/Workers/WorkerOptionsBuilder.cs b/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs similarity index 96% rename from src/Blumchen/Workers/WorkerOptionsBuilder.cs rename to src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs index f146b75..07c79c5 100644 --- a/src/Blumchen/Workers/WorkerOptionsBuilder.cs +++ b/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs @@ -1,7 +1,7 @@ using Blumchen.Subscriptions; using Polly; -namespace Blumchen.Workers; +namespace Blumchen.DependencyInjection; public record WorkerOptions(ResiliencePipeline ResiliencePipeline, ISubscriptionOptions SubscriptionOptions); diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index 735b15d..2c13d81 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using Blumchen.DependencyInjection; using Blumchen.Serialization; using Blumchen.Subscriptions; using Commons; @@ -10,7 +11,6 @@ using SubscriberWorker; using Npgsql; using Blumchen.Subscriptions.Management; -using Blumchen.Workers; using Polly.Registry; From f9bf48f7c57240506f700b653d731b6aa15695f0 Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 17 Jul 2024 10:10:26 +0200 Subject: [PATCH 22/80] mark as implicit usage --- src/Blumchen/MessageTableOptions.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Blumchen/MessageTableOptions.cs b/src/Blumchen/MessageTableOptions.cs index b038de3..ec91b9b 100644 --- a/src/Blumchen/MessageTableOptions.cs +++ b/src/Blumchen/MessageTableOptions.cs @@ -1,4 +1,5 @@ using Blumchen.Subscriptions; +using JetBrains.Annotations; using NpgsqlTypes; namespace Blumchen; @@ -33,6 +34,7 @@ public TableDescriptorBuilder MessageType(string name, int dimension = 250) return this; } + [UsedImplicitly] public TableDescriptorBuilder UseDefaults() => this; public record MessageTable(string Name = MessageTable.DefaultName) From 7fff1b4f8dbef35b6893c4e4a6e3b1223ebadba8 Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 17 Jul 2024 10:11:46 +0200 Subject: [PATCH 23/80] switch to prepared statement --- src/Blumchen/Publications/MessageAppender.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Blumchen/Publications/MessageAppender.cs b/src/Blumchen/Publications/MessageAppender.cs index c927dfe..4c4054f 100644 --- a/src/Blumchen/Publications/MessageAppender.cs +++ b/src/Blumchen/Publications/MessageAppender.cs @@ -35,11 +35,13 @@ private static async Task AppendAsyncOfT(T input { var (typeName, jsonTypeInfo) = typeResolver.Resolve(typeof(T)); var data = JsonSerialization.ToJson(@input, jsonTypeInfo); - var command = new NpgsqlCommand( + + await using var command = new NpgsqlCommand( $"INSERT INTO {tableDescriptor.Name}({tableDescriptor.MessageType.Name}, {tableDescriptor.Data.Name}) values ('{typeName}', '{data}')", connection, transaction ); + await command.PrepareAsync(ct).ConfigureAwait(false); await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false); } From f0c41607e5e976841d5a66e9df17db9cd1acc3a8 Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 17 Jul 2024 10:13:43 +0200 Subject: [PATCH 24/80] dispose resources --- src/Blumchen/Publications/MessageAppender.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Blumchen/Publications/MessageAppender.cs b/src/Blumchen/Publications/MessageAppender.cs index 4c4054f..66b7fdd 100644 --- a/src/Blumchen/Publications/MessageAppender.cs +++ b/src/Blumchen/Publications/MessageAppender.cs @@ -55,12 +55,12 @@ public static async Task AppendAsync(T input var (typeName, jsonTypeInfo) = options.resolver.Resolve(type); var data = JsonSerialization.ToJson(input, jsonTypeInfo); - var connection = new NpgsqlConnection(connectionString); - await using var connection1 = connection.ConfigureAwait(false); + await using var connection = new NpgsqlConnection(connectionString); await connection.OpenAsync(ct).ConfigureAwait(false); - var command = connection.CreateCommand(); + await using var command = connection.CreateCommand(); command.CommandText = $"INSERT INTO {options.tableDescriptor.Name}({options.tableDescriptor.MessageType.Name}, {options.tableDescriptor.Data.Name}) values ('{typeName}', '{data}')"; + await command.PrepareAsync(ct).ConfigureAwait(false); await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false); } @@ -71,7 +71,7 @@ private static async Task AppendBatchAsyncOfT(T inputs , NpgsqlTransaction transaction , CancellationToken ct) where T : class, IEnumerable { - var batch = new NpgsqlBatch(connection, transaction); + await using var batch = new NpgsqlBatch(connection, transaction); foreach (var input in inputs) { var (typeName, jsonTypeInfo) = resolver.Resolve(input.GetType()); From 8db4187b21d63b94724c7d67b06264bd2fb0ca9a Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 17 Jul 2024 10:21:59 +0200 Subject: [PATCH 25/80] explicit defaults --- src/Publisher/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Publisher/Program.cs b/src/Publisher/Program.cs index 41da1a0..a29d944 100644 --- a/src/Publisher/Program.cs +++ b/src/Publisher/Program.cs @@ -14,6 +14,7 @@ var resolver = new PublisherSetupOptionsBuilder() .JsonContext(SourceGenerationContext.Default) .NamingPolicy(new AttributeNamingPolicy()) + .WithTable(builder => builder.UseDefaults())//default, but explicit .Build(); do From c446721f3f68c3c4d9a90f41df052c5fe010a9f2 Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 17 Jul 2024 10:23:23 +0200 Subject: [PATCH 26/80] add PublisherOptions --- src/Blumchen/Publications/MessageAppender.cs | 12 ++++++------ src/Blumchen/Publications/PublisherOptions.cs | 5 +++++ .../Publications/PublisherSetupOptionsBuilder.cs | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 src/Blumchen/Publications/PublisherOptions.cs diff --git a/src/Blumchen/Publications/MessageAppender.cs b/src/Blumchen/Publications/MessageAppender.cs index 66b7fdd..860d80b 100644 --- a/src/Blumchen/Publications/MessageAppender.cs +++ b/src/Blumchen/Publications/MessageAppender.cs @@ -7,7 +7,7 @@ namespace Blumchen.Publications; public static class MessageAppender { public static async Task AppendAsync(T @input - , (TableDescriptorBuilder.MessageTable tableDescriptor, IJsonTypeResolver jsonTypeResolver) resolver + , PublisherOptions resolver , NpgsqlConnection connection , NpgsqlTransaction transaction , CancellationToken ct @@ -18,10 +18,10 @@ public static async Task AppendAsync(T @input case null: throw new ArgumentNullException(nameof(@input)); case IEnumerable inputs: - await AppendBatchAsyncOfT(inputs, resolver.tableDescriptor, resolver.jsonTypeResolver, connection, transaction, ct).ConfigureAwait(false); + await AppendBatchAsyncOfT(inputs, resolver.TableDescriptor, resolver.JsonTypeResolver, connection, transaction, ct).ConfigureAwait(false); break; default: - await AppendAsyncOfT(input, resolver.tableDescriptor, resolver.jsonTypeResolver, connection, transaction, ct).ConfigureAwait(false); + await AppendAsyncOfT(input, resolver.TableDescriptor, resolver.JsonTypeResolver, connection, transaction, ct).ConfigureAwait(false); break; } } @@ -46,20 +46,20 @@ private static async Task AppendAsyncOfT(T input } public static async Task AppendAsync(T input - , (TableDescriptorBuilder.MessageTable tableDescriptor, IJsonTypeResolver resolver) options + , PublisherOptions options , string connectionString , CancellationToken ct) where T: class { var type = typeof(T); - var (typeName, jsonTypeInfo) = options.resolver.Resolve(type); + var (typeName, jsonTypeInfo) = options.JsonTypeResolver.Resolve(type); var data = JsonSerialization.ToJson(input, jsonTypeInfo); await using var connection = new NpgsqlConnection(connectionString); await connection.OpenAsync(ct).ConfigureAwait(false); await using var command = connection.CreateCommand(); command.CommandText = - $"INSERT INTO {options.tableDescriptor.Name}({options.tableDescriptor.MessageType.Name}, {options.tableDescriptor.Data.Name}) values ('{typeName}', '{data}')"; + $"INSERT INTO {options.TableDescriptor.Name}({options.TableDescriptor.MessageType.Name}, {options.TableDescriptor.Data.Name}) values ('{typeName}', '{data}')"; await command.PrepareAsync(ct).ConfigureAwait(false); await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false); } diff --git a/src/Blumchen/Publications/PublisherOptions.cs b/src/Blumchen/Publications/PublisherOptions.cs new file mode 100644 index 0000000..d6bdeed --- /dev/null +++ b/src/Blumchen/Publications/PublisherOptions.cs @@ -0,0 +1,5 @@ +using Blumchen.Serialization; + +namespace Blumchen.Publications; + +public record PublisherOptions(TableDescriptorBuilder.MessageTable TableDescriptor, IJsonTypeResolver JsonTypeResolver); diff --git a/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs b/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs index dbd7095..e42c4a2 100644 --- a/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs +++ b/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs @@ -33,7 +33,7 @@ public PublisherSetupOptionsBuilder WithTable(Func Date: Wed, 17 Jul 2024 10:45:19 +0200 Subject: [PATCH 27/80] Expose shortcut for table validation when bootstrapping publisher --- src/Blumchen/Publications/PublisherOptions.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/Blumchen/Publications/PublisherOptions.cs b/src/Blumchen/Publications/PublisherOptions.cs index d6bdeed..d25c048 100644 --- a/src/Blumchen/Publications/PublisherOptions.cs +++ b/src/Blumchen/Publications/PublisherOptions.cs @@ -1,5 +1,20 @@ +using Blumchen.Database; using Blumchen.Serialization; +using Npgsql; namespace Blumchen.Publications; public record PublisherOptions(TableDescriptorBuilder.MessageTable TableDescriptor, IJsonTypeResolver JsonTypeResolver); + +public static class PublisherOptionsExtensions +{ + public static async Task EnsureTable(this PublisherOptions publisherOptions, NpgsqlDataSource dataSource, CancellationToken ct) + { + await dataSource.EnsureTableExists(publisherOptions.TableDescriptor, ct); + return publisherOptions; + } + + public static Task EnsureTable(this PublisherOptions publisherOptions, + string connectionString, CancellationToken ct) + => EnsureTable(publisherOptions, new NpgsqlDataSourceBuilder(connectionString).Build(), ct); +} From 2351af18bccd92b84de778aa49a06788bc8e68a5 Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 17 Jul 2024 10:46:25 +0200 Subject: [PATCH 28/80] explain different available to publisher for validating table --- src/Publisher/Program.cs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/Publisher/Program.cs b/src/Publisher/Program.cs index a29d944..cdbccc9 100644 --- a/src/Publisher/Program.cs +++ b/src/Publisher/Program.cs @@ -1,3 +1,4 @@ +using Blumchen.Database; using Blumchen.Publications; using Blumchen.Serialization; using Commons; @@ -10,12 +11,7 @@ Console.Title = typeof(Program).Assembly.GetName().Name!; Console.WriteLine("How many messages do you want to publish?(press CTRL+C to exit):"); - -var resolver = new PublisherSetupOptionsBuilder() - .JsonContext(SourceGenerationContext.Default) - .NamingPolicy(new AttributeNamingPolicy()) - .WithTable(builder => builder.UseDefaults())//default, but explicit - .Build(); +var cts = new CancellationTokenSource(); do { @@ -23,7 +19,19 @@ var line = Console.ReadLine(); if (line != null && int.TryParse(line, out var result)) { - var cts = new CancellationTokenSource(); + var resolver = await new PublisherSetupOptionsBuilder() + .JsonContext(SourceGenerationContext.Default) + .NamingPolicy(new AttributeNamingPolicy()) + .WithTable(builder => builder.UseDefaults()) //default, but explicit + .Build() + .EnsureTable(Settings.ConnectionString, cts.Token)//enforce table existence and conformity - db roundtrip + .ConfigureAwait(false); + + //Or you might want to verify at a later stage + await new NpgsqlDataSourceBuilder(Settings.ConnectionString) + .Build() + .EnsureTableExists(resolver.TableDescriptor, cts.Token).ConfigureAwait(false); + var messages = result / 4; var ct = cts.Token; var connection = new NpgsqlConnection(Settings.ConnectionString); From 9d3249a0c39139478dfa0122ecbbc3a77b24e4e1 Mon Sep 17 00:00:00 2001 From: giordanol Date: Sat, 6 Jul 2024 14:24:55 +0200 Subject: [PATCH 29/80] file renamed --- .../{IDictionaryExtensions.cs => JsonTypeResolverExtensions.cs} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/Blumchen/Serialization/{IDictionaryExtensions.cs => JsonTypeResolverExtensions.cs} (87%) diff --git a/src/Blumchen/Serialization/IDictionaryExtensions.cs b/src/Blumchen/Serialization/JsonTypeResolverExtensions.cs similarity index 87% rename from src/Blumchen/Serialization/IDictionaryExtensions.cs rename to src/Blumchen/Serialization/JsonTypeResolverExtensions.cs index 0017b82..3887b72 100644 --- a/src/Blumchen/Serialization/IDictionaryExtensions.cs +++ b/src/Blumchen/Serialization/JsonTypeResolverExtensions.cs @@ -1,7 +1,7 @@ namespace Blumchen.Serialization; -public static class DictionaryExtensions +public static class JsonTypeResolverExtensions { internal static IEnumerable Keys(this JsonTypeResolver? resolver) => resolver?.RegisteredTypes.Keys ?? Enumerable.Empty(); internal static IEnumerable Values(this JsonTypeResolver? resolver) => resolver?.RegisteredTypes.Values ?? Enumerable.Empty(); From ea6f7fe1a46454cb7c20df9539bf03bf8264e3a6 Mon Sep 17 00:00:00 2001 From: giordanol Date: Sun, 7 Jul 2024 14:20:22 +0200 Subject: [PATCH 30/80] renamed IHandler to IMessageHandler --- .../Blumchen.DependencyInjection.csproj | 54 ------------------- .../ServiceCollectionExtensions.cs | 4 +- src/Blumchen/DependencyInjection/Worker.cs | 2 +- src/Blumchen/Subscriptions/IHandler.cs | 8 --- src/Blumchen/Subscriptions/IMessageHandler.cs | 8 +++ .../Subscriptions/ISubscriptionOptions.cs | 4 +- .../Subscriptions/ObjectTracingConsumer.cs | 2 +- src/Blumchen/Subscriptions/Subscription.cs | 22 ++++---- .../SubscriptionOptionsBuilder.cs | 6 ++- src/Subscriber/Program.cs | 4 +- .../{Handler.cs => MessageHandler.cs} | 13 ++--- src/SubscriberWorker/Program.cs | 23 ++++---- src/Tests/DatabaseFixture.cs | 7 ++- 13 files changed, 50 insertions(+), 107 deletions(-) delete mode 100644 src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj delete mode 100644 src/Blumchen/Subscriptions/IHandler.cs create mode 100644 src/Blumchen/Subscriptions/IMessageHandler.cs rename src/SubscriberWorker/{Handler.cs => MessageHandler.cs} (79%) diff --git a/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj b/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj deleted file mode 100644 index 5c7b408..0000000 --- a/src/Blumchen.DependencyInjection/Blumchen.DependencyInjection.csproj +++ /dev/null @@ -1,54 +0,0 @@ - - - - 0.1.1 - net8.0 - true - true - true - false - true - true - true - true - 12.0 - Oskar Dudycz - - https://github.com/event-driven-io/Blumchen - MIT - https://github.com/event-driven-io/Blumchen.git - true - Blumchen - true - true - true - snupkg - Blumchen - true - enable - enable - - - - 1591 - - - - 1591 - - - - - all - none - all - - - - - - - - - - diff --git a/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs index c01d5ab..1cc6d55 100644 --- a/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs @@ -12,12 +12,12 @@ public static class ServiceCollectionExtensions public static IServiceCollection AddBlumchen( this IServiceCollection service, Func workerOptions) - where T : class, IHandler => + where T : class, IMessageHandler => service .AddKeyedSingleton(typeof(T), (provider, _) => workerOptions(provider, new WorkerOptionsBuilder()).Build()) .AddHostedService(provider => new Worker(workerOptions(provider, new WorkerOptionsBuilder()).Build(), - ServiceProviderServiceExtensions.GetRequiredService>>(provider))); + provider.GetRequiredService>>())); } diff --git a/src/Blumchen/DependencyInjection/Worker.cs b/src/Blumchen/DependencyInjection/Worker.cs index 4f137b7..4b670ff 100644 --- a/src/Blumchen/DependencyInjection/Worker.cs +++ b/src/Blumchen/DependencyInjection/Worker.cs @@ -7,7 +7,7 @@ namespace Blumchen.DependencyInjection; public class Worker( WorkerOptions options, - ILogger> logger): BackgroundService where T : class, IHandler + ILogger> logger): BackgroundService where T : class, IMessageHandler { private string WorkerName { get; } = $"{nameof(Worker)}<{typeof(T).Name}>"; private static readonly ConcurrentDictionary> LoggingActions = new(StringComparer.OrdinalIgnoreCase); diff --git a/src/Blumchen/Subscriptions/IHandler.cs b/src/Blumchen/Subscriptions/IHandler.cs deleted file mode 100644 index b722f25..0000000 --- a/src/Blumchen/Subscriptions/IHandler.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Blumchen.Subscriptions; - -public interface IHandler; - -public interface IHandler: IHandler where T : class -{ - Task Handle(T value); -} diff --git a/src/Blumchen/Subscriptions/IMessageHandler.cs b/src/Blumchen/Subscriptions/IMessageHandler.cs new file mode 100644 index 0000000..3a55d10 --- /dev/null +++ b/src/Blumchen/Subscriptions/IMessageHandler.cs @@ -0,0 +1,8 @@ +namespace Blumchen.Subscriptions; + +public interface IMessageHandler; + +public interface IMessageHandler: IMessageHandler where T : class +{ + Task Handle(T value); +} diff --git a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs b/src/Blumchen/Subscriptions/ISubscriptionOptions.cs index 27dd513..14b75a4 100644 --- a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs +++ b/src/Blumchen/Subscriptions/ISubscriptionOptions.cs @@ -22,7 +22,7 @@ void Deconstruct( out ReplicationSlotSetupOptions replicationSlotSetupOptions, out IErrorProcessor errorProcessor, out IReplicationDataMapper dataMapper, - out Dictionary registry); + out Dictionary registry); } internal record SubscriptionOptions( @@ -32,4 +32,4 @@ internal record SubscriptionOptions( ReplicationSlotSetupOptions ReplicationOptions, IErrorProcessor ErrorProcessor, IReplicationDataMapper DataMapper, - Dictionary Registry): ISubscriptionOptions; + Dictionary Registry): ISubscriptionOptions; diff --git a/src/Blumchen/Subscriptions/ObjectTracingConsumer.cs b/src/Blumchen/Subscriptions/ObjectTracingConsumer.cs index 3263482..f3d65b8 100644 --- a/src/Blumchen/Subscriptions/ObjectTracingConsumer.cs +++ b/src/Blumchen/Subscriptions/ObjectTracingConsumer.cs @@ -1,6 +1,6 @@ namespace Blumchen.Subscriptions; -internal class ObjectTracingConsumer: IHandler +internal class ObjectTracingConsumer: IMessageHandler { private static ulong _counter = 0; public Task Handle(object value) diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 872df0b..4a4defd 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -91,7 +91,7 @@ internal async IAsyncEnumerable Subscribe( private static async IAsyncEnumerable ProcessEnvelope( IEnvelope envelope, - Dictionary registry, + Dictionary registry, IErrorProcessor errorProcessor ) where T:class { @@ -104,22 +104,22 @@ IErrorProcessor errorProcessor { var obj = okEnvelope.Value; var objType = obj.GetType(); - var (consumer, methodInfo) = Memoize(registry, objType, Consumer); - await ((Task)methodInfo.Invoke(consumer, [obj])!).ConfigureAwait(false); + var (messageHandler, methodInfo) = Memoize(registry, objType, MessageHandler); + await ((Task)methodInfo.Invoke(messageHandler, [obj])!).ConfigureAwait(false); yield return (T)envelope; yield break; } } } - private static readonly Dictionary Cache = []; + private static readonly Dictionary Cache = []; - private static (IHandler consumer, MethodInfo methodInfo) Memoize + private static (IMessageHandler messageHandler, MethodInfo methodInfo) Memoize ( - Dictionary registry, + Dictionary registry, Type objType, - Func, Type, (IHandler consumer, MethodInfo methodInfo)> func + Func, Type, (IMessageHandler messageHandler, MethodInfo methodInfo)> func ) { if (!Cache.TryGetValue(objType, out var entry)) @@ -127,13 +127,13 @@ private static (IHandler consumer, MethodInfo methodInfo) Memoize Cache[objType] = entry; return entry; } - private static (IHandler consumer, MethodInfo methodInfo) Consumer(Dictionary registry, Type objType) + private static (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler(Dictionary registry, Type objType) { - var consumer = registry[objType] ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); - var methodInfos = consumer.GetType().GetMethods(BindingFlags.Instance|BindingFlags.Public); + var messageHandler = registry[objType] ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); + var methodInfos = messageHandler.GetType().GetMethods(BindingFlags.Instance|BindingFlags.Public); var methodInfo = methodInfos.SingleOrDefault(mi=>mi.GetParameters().Any(pa => pa.ParameterType == objType)) ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); - return (consumer, methodInfo); + return (messageHandler, methodInfo); } private static async IAsyncEnumerable ReadExistingRowsFromSnapshot( diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index 8166ab2..78bc69e 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -9,12 +9,14 @@ namespace Blumchen.Subscriptions; public sealed class SubscriptionOptionsBuilder { + private NpgsqlConnectionStringBuilder? _connectionStringBuilder; private NpgsqlDataSource? _dataSource; private PublicationManagement.PublicationSetupOptions _publicationSetupOptions = new(); private ReplicationSlotManagement.ReplicationSlotSetupOptions? _replicationSlotSetupOptions; private IReplicationDataMapper? _dataMapper; - private readonly Dictionary _registry = []; + private readonly Dictionary _registry = []; + private IErrorProcessor? _errorProcessor; private INamingPolicy? _namingPolicy; private JsonSerializerContext? _jsonSerializerContext; @@ -73,7 +75,7 @@ public SubscriptionOptionsBuilder WithReplicationOptions(ReplicationSlotManageme } [UsedImplicitly] - public SubscriptionOptionsBuilder Consumes(IHandler handler) where T : class + public SubscriptionOptionsBuilder Consumes(IMessageHandler handler) where T : class { _registry.TryAdd(typeof(T), handler); return this; diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index 1d129f8..2ac77db 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -50,8 +50,8 @@ namespace Subscriber { internal class Consumer: - IHandler, - IHandler + IMessageHandler, + IMessageHandler { public Task Handle(UserCreatedContract value) => Console.Out.WriteLineAsync(JsonSerialization.ToJson(value, SourceGenerationContext.Default.UserCreatedContract)); public Task Handle(UserDeletedContract value) => Console.Out.WriteLineAsync(JsonSerialization.ToJson(value, SourceGenerationContext.Default.UserDeletedContract)); diff --git a/src/SubscriberWorker/Handler.cs b/src/SubscriberWorker/MessageHandler.cs similarity index 79% rename from src/SubscriberWorker/Handler.cs rename to src/SubscriberWorker/MessageHandler.cs index 3ecf3fb..9076b4a 100644 --- a/src/SubscriberWorker/Handler.cs +++ b/src/SubscriberWorker/MessageHandler.cs @@ -6,6 +6,9 @@ namespace SubscriberWorker; public class HandlerBase(ILogger logger) { + private int _counter; + private int _completed; + private Task ReportSuccess(int count) { if (logger.IsEnabled(LogLevel.Debug)) @@ -13,8 +16,6 @@ private Task ReportSuccess(int count) return Task.CompletedTask; } - private int _counter; - private int _completed; protected Task Handle(T value) => Interlocked.Increment(ref _counter) % 10 == 0 //Simulating some exception on out of process dependencies @@ -22,14 +23,14 @@ protected Task Handle(T value) => : ReportSuccess(Interlocked.Increment(ref _completed)); } -public class HandleImpl2(ILogger logger): HandlerBase(logger), - IHandler +public class HandleImpl2(ILogger logger) + : HandlerBase(logger), IMessageHandler { public Task Handle(UserDeletedContract value) => Handle(value); } -public class HandleImpl1(ILogger logger) : HandlerBase(logger), - IHandler, IHandler +public class HandleImpl1(ILogger logger) + : HandlerBase(logger), IMessageHandler, IMessageHandler { public Task Handle(UserCreatedContract value) => Handle(value); diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index 2c13d81..04f5de1 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -13,13 +13,10 @@ using Blumchen.Subscriptions.Management; using Polly.Registry; - #pragma warning disable CS8601 // Possible null reference assignment. Console.Title = typeof(Program).Assembly.GetName().Name; #pragma warning restore CS8601 // Possible null reference assignment. - - AppDomain.CurrentDomain.UnhandledException += (_, e) => Console.Out.WriteLine(e.ExceptionObject.ToString()); TaskScheduler.UnobservedTaskException += (_, e) => Console.Out.WriteLine(e.Exception.ToString()); @@ -27,10 +24,9 @@ var builder = Host.CreateApplicationBuilder(args); builder.Services - - .AddSingleton, HandleImpl1>() - .AddSingleton, HandleImpl1>() - .AddSingleton, HandleImpl2>() + .AddSingleton, HandleImpl1>() + .AddSingleton, HandleImpl1>() + .AddSingleton, HandleImpl2>() .AddSingleton() @@ -71,12 +67,11 @@ .WithErrorProcessor(provider.GetRequiredService()) .NamingPolicy(provider.GetRequiredService()) .JsonContext(SourceGenerationContext.Default) - .Consumes(provider.GetRequiredService>()) - .Consumes(provider.GetRequiredService>())) + .Consumes(provider.GetRequiredService>()) + .Consumes(provider.GetRequiredService>()) + ) .ResiliencyPipeline(provider.GetRequiredService>().GetPipeline("default")) - ) - .AddBlumchen((provider, workerOptions) => workerOptions .Subscription(subscriptionOptions => @@ -87,10 +82,10 @@ .WithErrorProcessor(provider.GetRequiredService()) .NamingPolicy(provider.GetRequiredService()) .JsonContext(SourceGenerationContext.Default) - .Consumes(provider.GetRequiredService>())) + .Consumes(provider.GetRequiredService>()) + ) .ResiliencyPipeline(provider.GetRequiredService>().GetPipeline("default")) - ) - ; + ); await builder .Build() diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index 1b2f1af..c77e5a6 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -12,12 +12,11 @@ namespace Tests; - public abstract class DatabaseFixture(ITestOutputHelper output): IAsyncLifetime { protected ITestOutputHelper Output { get; } = output; protected readonly Func TimeoutTokenSource = () => new(Debugger.IsAttached ? TimeSpan.FromHours(1) : TimeSpan.FromSeconds(2)); - protected class TestHandler(Action log, JsonTypeInfo info): IHandler where T : class + protected class TestMessageHandler(Action log, JsonTypeInfo info): IMessageHandler where T : class { public async Task Handle(T value) { @@ -67,7 +66,7 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri await command.ExecuteNonQueryAsync(ct); } - protected (TestHandler handler, SubscriptionOptionsBuilder subscriptionOptionsBuilder) SetupFor( + protected (TestMessageHandler handler, SubscriptionOptionsBuilder subscriptionOptionsBuilder) SetupFor( string connectionString, string eventsTable, JsonSerializerContext info, @@ -78,7 +77,7 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri { var jsonTypeInfo = info.GetTypeInfo(typeof(T)); ArgumentNullException.ThrowIfNull(jsonTypeInfo); - var consumer = new TestHandler(log, jsonTypeInfo); + var consumer = new TestMessageHandler(log, jsonTypeInfo); var subscriptionOptionsBuilder = new SubscriptionOptionsBuilder() .WithErrorProcessor(new TestOutErrorProcessor(Output)) .DataSource(new NpgsqlDataSourceBuilder(connectionString).Build()) From 3887f148e0dad88c2b057cab9b6b822a633fca14 Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 17 Jul 2024 19:10:00 +0200 Subject: [PATCH 31/80] first working version --- src/Blumchen/Database/Run.cs | 2 +- src/Blumchen/Publications/MessageAppender.cs | 5 +- src/Blumchen/Publications/PublisherOptions.cs | 3 +- .../PublisherSetupOptionsBuilder.cs | 1 + src/Blumchen/Serialization/ITypeResolver.cs | 10 +- .../JsonTypeResolverExtensions.cs | 4 +- .../Serialization/MessageUrnAttribute.cs | 44 +++++- .../Subscriptions/ISubscriptionOptions.cs | 8 +- .../Management/PublicationManagement.cs | 29 ++-- .../Replication/IReplicationDataMapper.cs | 2 + .../Replication/ReplicationDataMapper.cs | 134 ++++++++++++++++-- .../ReplicationMessageHandlers/Envelope.cs | 2 +- .../SnapshotReader/SnapshotReader.cs | 2 +- src/Blumchen/Subscriptions/Subscription.cs | 65 +++++---- .../SubscriptionOptionsBuilder.cs | 78 ++++++---- src/Subscriber/Contracts.cs | 12 +- src/Subscriber/Program.cs | 14 +- 17 files changed, 300 insertions(+), 115 deletions(-) diff --git a/src/Blumchen/Database/Run.cs b/src/Blumchen/Database/Run.cs index f4ccec1..fd1d06a 100644 --- a/src/Blumchen/Database/Run.cs +++ b/src/Blumchen/Database/Run.cs @@ -40,7 +40,7 @@ internal static async IAsyncEnumerable QueryTransactionSnapshot(this string snapshotName, TableDescriptorBuilder.MessageTable tableDescriptor, ISet registeredTypesKeys, - IReplicationDataMapper dataMapper, + ReplicationDataMapper dataMapper, [EnumeratorCancellation] CancellationToken ct) { var transaction = await connection.BeginTransactionAsync(IsolationLevel.RepeatableRead, ct).ConfigureAwait(false); diff --git a/src/Blumchen/Publications/MessageAppender.cs b/src/Blumchen/Publications/MessageAppender.cs index 860d80b..5fd963c 100644 --- a/src/Blumchen/Publications/MessageAppender.cs +++ b/src/Blumchen/Publications/MessageAppender.cs @@ -1,4 +1,5 @@ using System.Collections; +using System.Text.Json.Serialization.Metadata; using Blumchen.Serialization; using Npgsql; @@ -28,7 +29,7 @@ public static async Task AppendAsync(T @input private static async Task AppendAsyncOfT(T input , TableDescriptorBuilder.MessageTable tableDescriptor - , IJsonTypeResolver typeResolver + , ITypeResolver typeResolver , NpgsqlConnection connection , NpgsqlTransaction transaction , CancellationToken ct) where T : class @@ -66,7 +67,7 @@ public static async Task AppendAsync(T input private static async Task AppendBatchAsyncOfT(T inputs , TableDescriptorBuilder.MessageTable tableDescriptor - , IJsonTypeResolver resolver + , ITypeResolver resolver , NpgsqlConnection connection , NpgsqlTransaction transaction , CancellationToken ct) where T : class, IEnumerable diff --git a/src/Blumchen/Publications/PublisherOptions.cs b/src/Blumchen/Publications/PublisherOptions.cs index d25c048..9c16d3f 100644 --- a/src/Blumchen/Publications/PublisherOptions.cs +++ b/src/Blumchen/Publications/PublisherOptions.cs @@ -1,10 +1,11 @@ +using System.Text.Json.Serialization.Metadata; using Blumchen.Database; using Blumchen.Serialization; using Npgsql; namespace Blumchen.Publications; -public record PublisherOptions(TableDescriptorBuilder.MessageTable TableDescriptor, IJsonTypeResolver JsonTypeResolver); +public record PublisherOptions(TableDescriptorBuilder.MessageTable TableDescriptor, ITypeResolver JsonTypeResolver); public static class PublisherOptionsExtensions { diff --git a/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs b/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs index e42c4a2..00be8f1 100644 --- a/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs +++ b/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; using Blumchen.Serialization; using JetBrains.Annotations; using static Blumchen.TableDescriptorBuilder; diff --git a/src/Blumchen/Serialization/ITypeResolver.cs b/src/Blumchen/Serialization/ITypeResolver.cs index cdddbf4..4695d9c 100644 --- a/src/Blumchen/Serialization/ITypeResolver.cs +++ b/src/Blumchen/Serialization/ITypeResolver.cs @@ -7,14 +7,14 @@ namespace Blumchen.Serialization; public interface ITypeResolver { (string, T) Resolve(Type type); + Type Resolve(string type); + IDictionary RegisteredTypes { get; } } -public interface IJsonTypeResolver: ITypeResolver; - internal sealed class JsonTypeResolver( JsonSerializerContext serializationContext, INamingPolicy? namingPolicy = default) - : IJsonTypeResolver + : ITypeResolver { public JsonSerializerContext SerializationContext { get; } = serializationContext; private readonly ConcurrentDictionary _typeDictionary = []; @@ -31,7 +31,7 @@ internal void WhiteList(Type type) public (string, JsonTypeInfo) Resolve(Type type) => (_typeDictionary.Single(kv => kv.Value == type).Key, _typeInfoDictionary[type]); - internal IDictionary RegisteredTypes { get => _typeDictionary; } - internal Type Resolve(string type) => _typeDictionary[type]; + public IDictionary RegisteredTypes { get => _typeDictionary; } + public Type Resolve(string type) => _typeDictionary[type]; } diff --git a/src/Blumchen/Serialization/JsonTypeResolverExtensions.cs b/src/Blumchen/Serialization/JsonTypeResolverExtensions.cs index 3887b72..2d692cd 100644 --- a/src/Blumchen/Serialization/JsonTypeResolverExtensions.cs +++ b/src/Blumchen/Serialization/JsonTypeResolverExtensions.cs @@ -3,6 +3,6 @@ namespace Blumchen.Serialization; public static class JsonTypeResolverExtensions { - internal static IEnumerable Keys(this JsonTypeResolver? resolver) => resolver?.RegisteredTypes.Keys ?? Enumerable.Empty(); - internal static IEnumerable Values(this JsonTypeResolver? resolver) => resolver?.RegisteredTypes.Values ?? Enumerable.Empty(); + internal static IEnumerable Keys(this ITypeResolver? resolver) => resolver?.RegisteredTypes.Keys ?? Enumerable.Empty(); + internal static IEnumerable Values(this ITypeResolver? resolver) => resolver?.RegisteredTypes.Values ?? Enumerable.Empty(); } diff --git a/src/Blumchen/Serialization/MessageUrnAttribute.cs b/src/Blumchen/Serialization/MessageUrnAttribute.cs index 11ae685..f62fe42 100644 --- a/src/Blumchen/Serialization/MessageUrnAttribute.cs +++ b/src/Blumchen/Serialization/MessageUrnAttribute.cs @@ -2,7 +2,7 @@ namespace Blumchen.Serialization; -[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface)] +[AttributeUsage(AttributeTargets.Class)] public class MessageUrnAttribute: Attribute { @@ -33,6 +33,48 @@ private static Uri FormatUrn(string urn) } +public enum RawData +{ + String, + Object +} + +[AttributeUsage(AttributeTargets.Interface, AllowMultiple = true)] +public class RawUrnAttribute: + Attribute +{ + public RawData Data { get; } + + /// + /// + /// The urn value to use for this message type. + /// The value to bind to this + public RawUrnAttribute(string urn, RawData data) + { + Data = data; + ArgumentException.ThrowIfNullOrEmpty(urn, nameof(urn)); + + if (urn.StartsWith(MessageUrn.Prefix)) + throw new ArgumentException($"Value should not contain the default prefix '{MessageUrn.Prefix}'.", nameof(urn)); + + Urn = FormatUrn(urn); + } + + public Uri Urn { get; } + + private static Uri FormatUrn(string urn) + { + var fullValue = MessageUrn.Prefix + urn; + + if (Uri.TryCreate(fullValue, UriKind.Absolute, out var uri)) + return uri; + + throw new UriFormatException($"Invalid URN: {fullValue}"); + } +} + + + public static class MessageUrn { public const string Prefix = "urn:message:"; diff --git a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs b/src/Blumchen/Subscriptions/ISubscriptionOptions.cs index 14b75a4..74aca0e 100644 --- a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs +++ b/src/Blumchen/Subscriptions/ISubscriptionOptions.cs @@ -10,7 +10,7 @@ public interface ISubscriptionOptions { [UsedImplicitly] NpgsqlDataSource DataSource { get; } [UsedImplicitly] NpgsqlConnectionStringBuilder ConnectionStringBuilder { get; } - IReplicationDataMapper DataMapper { get; } + IDictionary> Registry { get; } [UsedImplicitly] PublicationSetupOptions PublicationOptions { get; } [UsedImplicitly] ReplicationSlotSetupOptions ReplicationOptions { get; } [UsedImplicitly] IErrorProcessor ErrorProcessor { get; } @@ -21,8 +21,7 @@ void Deconstruct( out PublicationSetupOptions publicationSetupOptions, out ReplicationSlotSetupOptions replicationSlotSetupOptions, out IErrorProcessor errorProcessor, - out IReplicationDataMapper dataMapper, - out Dictionary registry); + out IDictionary> registry); } internal record SubscriptionOptions( @@ -31,5 +30,4 @@ internal record SubscriptionOptions( PublicationSetupOptions PublicationOptions, ReplicationSlotSetupOptions ReplicationOptions, IErrorProcessor ErrorProcessor, - IReplicationDataMapper DataMapper, - Dictionary Registry): ISubscriptionOptions; + IDictionary> Registry): ISubscriptionOptions; diff --git a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs index c2b6f6c..baf1bde 100644 --- a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs +++ b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs @@ -1,3 +1,4 @@ +using System.Text.Json.Serialization.Metadata; using Blumchen.Database; using Blumchen.Serialization; using Npgsql; @@ -16,14 +17,14 @@ public static async Task SetupPublication( CancellationToken ct ) { - var (publicationName, createStyle, shouldReAddTablesIfWereRecreated, typeResolver, tableDescription) = setupOptions; + var (publicationName, createStyle, shouldReAddTablesIfWereRecreated, registeredTypes, tableDescription) = setupOptions; return createStyle switch { Subscription.CreateStyle.Never => new None(), - Subscription.CreateStyle.AlwaysRecreate => await ReCreate(dataSource, publicationName, tableDescription.Name, typeResolver, ct).ConfigureAwait(false), + Subscription.CreateStyle.AlwaysRecreate => await ReCreate(dataSource, publicationName, tableDescription.Name, registeredTypes, ct).ConfigureAwait(false), Subscription.CreateStyle.WhenNotExists when await dataSource.PublicationExists(publicationName, ct).ConfigureAwait(false) => await Refresh(dataSource, publicationName, tableDescription.Name, shouldReAddTablesIfWereRecreated, ct).ConfigureAwait(false), - Subscription.CreateStyle.WhenNotExists => await Create(dataSource, publicationName, tableDescription.Name, typeResolver, ct).ConfigureAwait(false), + Subscription.CreateStyle.WhenNotExists => await Create(dataSource, publicationName, tableDescription.Name, registeredTypes, ct).ConfigureAwait(false), _ => throw new ArgumentOutOfRangeException(nameof(setupOptions.CreateStyle)) }; @@ -31,22 +32,20 @@ static async Task ReCreate( NpgsqlDataSource dataSource, string publicationName, string tableName, - JsonTypeResolver? typeResolver, + ISet registeredTypes, CancellationToken ct ) { await dataSource.DropPublication(publicationName, ct).ConfigureAwait(false); - return await Create(dataSource, publicationName, tableName, typeResolver, ct).ConfigureAwait(false); + return await Create(dataSource, publicationName, tableName, registeredTypes, ct).ConfigureAwait(false); } static async Task Create(NpgsqlDataSource dataSource, string publicationName, string tableName, - JsonTypeResolver? typeResolver, + ISet registeredTypes, CancellationToken ct ) { - await dataSource.CreatePublication(publicationName, tableName, - typeResolver.Keys().ToHashSet(), ct).ConfigureAwait(false); - + await dataSource.CreatePublication(publicationName, tableName, registeredTypes, ct).ConfigureAwait(false); return new Created(); } @@ -66,14 +65,14 @@ internal static Task CreatePublication( this NpgsqlDataSource dataSource, string publicationName, string tableName, - ISet eventTypes, + ISet registeredTypes, CancellationToken ct ) { var sql = $"CREATE PUBLICATION \"{publicationName}\" FOR TABLE {tableName} {{0}} WITH (publish = 'insert');"; - return eventTypes.Count switch + return registeredTypes.Count switch { 0 => Execute(dataSource, string.Format(sql,string.Empty), ct), - _ => Execute(dataSource, string.Format(sql, $"WHERE ({PublicationFilter(eventTypes)})"), ct) + _ => Execute(dataSource, string.Format(sql, $"WHERE ({PublicationFilter(registeredTypes)})"), ct) }; static string PublicationFilter(ICollection input) => string.Join(" OR ", input.Select(s => $"message_type = '{s}'")); } @@ -143,7 +142,7 @@ public sealed record PublicationSetupOptions( ) { internal const string DefaultPublicationName = "pub"; - internal JsonTypeResolver? TypeResolver { get; init; } = default; + internal ISet RegisteredTypes { get; init; } = Enumerable.Empty().ToHashSet(); internal TableDescriptorBuilder.MessageTable TableDescriptor { get; init; } = new TableDescriptorBuilder().Build(); @@ -151,13 +150,13 @@ internal void Deconstruct( out string publicationName, out Subscription.CreateStyle createStyle, out bool reAddTablesIfWereRecreated, - out JsonTypeResolver? typeResolver, + out ISet registeredTypes, out TableDescriptorBuilder.MessageTable tableDescription) { publicationName = PublicationName; createStyle = Subscription.CreateStyle.WhenNotExists; reAddTablesIfWereRecreated = ShouldReAddTablesIfWereRecreated; - typeResolver = TypeResolver; + registeredTypes = RegisteredTypes; tableDescription = TableDescriptor; } diff --git a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs index 1f74c3a..a6ccd90 100644 --- a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs @@ -9,4 +9,6 @@ public interface IReplicationDataMapper Task ReadFromSnapshot(NpgsqlDataReader reader, CancellationToken ct); Task ReadFromReplication(InsertMessage insertMessage, CancellationToken ct); + + } diff --git a/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs index 75be676..52b8c91 100644 --- a/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Text.Json.Serialization.Metadata; using Blumchen.Serialization; using Blumchen.Subscriptions.ReplicationMessageHandlers; using Npgsql; @@ -7,7 +8,61 @@ namespace Blumchen.Subscriptions.Replication; -internal sealed class ReplicationDataMapper(JsonTypeResolver resolver): IReplicationDataMapper +internal interface IReplicationDataReader +{ + Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default); + Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default); +} + +internal class ObjectReplicationDataReader: IReplicationDataReader +{ + public Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default) + => replicationValue.Get(ct).AsTask(); + + + public Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default) + => reader.GetFieldValueAsync(2, ct); +} + + +internal class StringReplicationDataReader : IReplicationDataReader +{ + public async Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default) + { + using var tr = replicationValue.GetTextReader(); + return await tr.ReadToEndAsync(ct).ConfigureAwait(false); + } + + + public async Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default) + { + using var tr = await reader.GetTextReaderAsync(2, ct).ConfigureAwait(false); + return await tr.ReadToEndAsync(ct).ConfigureAwait(false); + } +} + +internal class JsonReplicationDataReader(JsonTypeResolver resolver): IReplicationDataReader +{ + public async Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default) + { + ArgumentNullException.ThrowIfNull(type); + await using var stream = replicationValue.GetStream(); + return await JsonSerialization.FromJsonAsync(type, stream, resolver.SerializationContext, ct) + .ConfigureAwait(false); + } + + public async Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default) + { + ArgumentNullException.ThrowIfNull(type); + var stream = await reader.GetStreamAsync(2, ct).ConfigureAwait(false); + await using var stream1 = stream.ConfigureAwait(false); + return await JsonSerialization.FromJsonAsync(type, stream, resolver.SerializationContext, ct) + .ConfigureAwait(false); + } +} + +internal class ReplicationDataMapper(IDictionary> mapperSelector) + : IReplicationDataMapper { public async Task ReadFromReplication(InsertMessage insertMessage, CancellationToken ct) { @@ -32,16 +87,12 @@ public async Task ReadFromReplication(InsertMessage insertMessage, Ca break; } case 2 when column.GetDataTypeName().Equals("jsonb", StringComparison.OrdinalIgnoreCase): - { - var type = resolver.Resolve(typeName); - ArgumentNullException.ThrowIfNull(type, typeName); - return new OkEnvelope(await JsonSerialization.FromJsonAsync(type, column.GetStream(), resolver.SerializationContext, ct).ConfigureAwait(false)); - } + return await mapperSelector[typeName].Item1.ReadFromReplication(id, typeName, column, ct); } } catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or JsonException) { - return new KoEnvelope(ex,id); + return new KoEnvelope(ex, id); } columnNumber++; } @@ -54,12 +105,9 @@ public async Task ReadFromSnapshot(NpgsqlDataReader reader, Cancellat try { id = reader.GetInt64(0); - var eventTypeName = reader.GetString(1); - var eventType = resolver.Resolve(eventTypeName); - ArgumentNullException.ThrowIfNull(eventType, eventTypeName); - var stream = await reader.GetStreamAsync(2, ct).ConfigureAwait(false); - await using var stream1 = stream.ConfigureAwait(false); - return new OkEnvelope(await JsonSerialization.FromJsonAsync(eventType, stream, resolver.SerializationContext, ct).ConfigureAwait(false)); + var typeName = reader.GetString(1); + + return await mapperSelector[typeName].Item1.ReadFromSnapshot(typeName, id, reader, ct); } catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or JsonException) { @@ -67,3 +115,63 @@ public async Task ReadFromSnapshot(NpgsqlDataReader reader, Cancellat } } } + +public interface IReplicationJsonBMapper +{ + Task ReadFromReplication(string id, string typeName, ReplicationValue column, + CancellationToken ct); + + Task ReadFromSnapshot(string typeName, long id, NpgsqlDataReader reader, CancellationToken ct); +} + +internal class ReplicationDataMapper( + IReplicationDataReader replicationDataReader + , ITypeResolver? resolver = default + ): IReplicationJsonBMapper +{ + public async Task ReadFromReplication(string id, string typeName, ReplicationValue column, + CancellationToken ct) + { + + try + { + var type = resolver?.Resolve(typeName); + var value = await replicationDataReader.Read(column, ct, type).ConfigureAwait(false) ?? + throw new ArgumentNullException(); + return new OkEnvelope(value, typeName); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException + or JsonException) + { + return new KoEnvelope(ex, id); + } + } + + public async Task ReadFromSnapshot(string typeName, long id, NpgsqlDataReader reader, CancellationToken ct) + { + try + { + var eventType = resolver?.Resolve(typeName); + ArgumentNullException.ThrowIfNull(eventType, typeName); + var value = await replicationDataReader.Read(reader, ct, eventType).ConfigureAwait(false) ?? throw new ArgumentNullException(); + return new OkEnvelope(value, typeName); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or JsonException) + { + return new KoEnvelope(ex, id.ToString()); + } + } +} + +internal sealed class ObjectReplicationDataMapper( + IReplicationDataReader replicationDataReader +): ReplicationDataMapper(replicationDataReader); + +internal sealed class StringReplicationDataMapper( + IReplicationDataReader replicationDataReader +): ReplicationDataMapper(replicationDataReader); + +internal sealed class JsonReplicationDataMapper( + ITypeResolver resolver, + IReplicationDataReader replicationDataReader +): ReplicationDataMapper(replicationDataReader, resolver); diff --git a/src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs b/src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs index aa2e812..9cd39da 100644 --- a/src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs +++ b/src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs @@ -2,6 +2,6 @@ namespace Blumchen.Subscriptions.ReplicationMessageHandlers; public interface IEnvelope; -public sealed record OkEnvelope(object Value): IEnvelope; +public sealed record OkEnvelope(object Value, string MessageType): IEnvelope; public sealed record KoEnvelope(Exception Error, string Id): IEnvelope; diff --git a/src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs b/src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs index eaaffe4..4e01ca4 100644 --- a/src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs +++ b/src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs @@ -11,7 +11,7 @@ public static class SnapshotReader internal static async IAsyncEnumerable GetRowsFromSnapshot(this NpgsqlConnection connection, string snapshotName, TableDescriptorBuilder.MessageTable tableDescriptor, - IReplicationDataMapper dataMapper, + ReplicationDataMapper dataMapper, ISet registeredTypes, [EnumeratorCancellation] CancellationToken ct = default) { diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 4a4defd..bc10dc8 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -1,8 +1,8 @@ using System.Reflection; using System.Runtime.CompilerServices; using Blumchen.Database; -using Blumchen.Serialization; using Blumchen.Subscriptions.Management; +using Blumchen.Subscriptions.Replication; using Blumchen.Subscriptions.ReplicationMessageHandlers; using Blumchen.Subscriptions.SnapshotReader; using Npgsql; @@ -26,7 +26,7 @@ public enum CreateStyle } private LogicalReplicationConnection? _connection; private readonly SubscriptionOptionsBuilder _builder = new(); - private ISubscriptionOptions? _options; + public async IAsyncEnumerable Subscribe( Func builder, [EnumeratorCancellation] CancellationToken ct = default @@ -41,9 +41,7 @@ internal async IAsyncEnumerable Subscribe( [EnumeratorCancellation] CancellationToken ct = default ) { - _options = subscriptionOptions; - var (dataSource, connectionStringBuilder, publicationSetupOptions, replicationSlotSetupOptions, errorProcessor, replicationDataMapper, registry) = subscriptionOptions; - + var (dataSource, connectionStringBuilder, publicationSetupOptions, replicationSlotSetupOptions, errorProcessor, registry) = subscriptionOptions; await dataSource.EnsureTableExists(publicationSetupOptions.TableDescriptor, ct).ConfigureAwait(false); _connection = new LogicalReplicationConnection(connectionStringBuilder.ConnectionString); @@ -51,7 +49,7 @@ internal async IAsyncEnumerable Subscribe( await dataSource.SetupPublication(publicationSetupOptions, ct).ConfigureAwait(false); var result = await dataSource.SetupReplicationSlot(_connection, replicationSlotSetupOptions, ct).ConfigureAwait(false); - + var replicationDataMapper = new ReplicationDataMapper(registry); PgOutputReplicationSlot slot; if (result is not Created created) @@ -67,9 +65,11 @@ internal async IAsyncEnumerable Subscribe( ) ); - await foreach (var envelope in ReadExistingRowsFromSnapshot(dataSource, created.SnapshotName, _options, ct).ConfigureAwait(false)) - await foreach (var subscribe in ProcessEnvelope(envelope, registry, errorProcessor).WithCancellation(ct).ConfigureAwait(false)) + await foreach (var envelope in ReadExistingRowsFromSnapshot(dataSource, created.SnapshotName, replicationDataMapper, publicationSetupOptions.TableDescriptor, publicationSetupOptions.RegisteredTypes, ct).ConfigureAwait(false)) + { + await foreach (var subscribe in ProcessEnvelope(envelope, registry, errorProcessor).WithCancellation(ct).ConfigureAwait(false)) yield return subscribe; + } } await foreach (var message in @@ -79,7 +79,7 @@ internal async IAsyncEnumerable Subscribe( if (message is InsertMessage insertMessage) { var envelope = await replicationDataMapper.ReadFromReplication(insertMessage, ct).ConfigureAwait(false); - await foreach (var subscribe in ProcessEnvelope(envelope, registry, errorProcessor).WithCancellation(ct).ConfigureAwait(false)) + await foreach (var subscribe in ProcessEnvelope(envelope, registry, errorProcessor).WithCancellation(ct).ConfigureAwait(false)) yield return subscribe; } // Always call SetReplicationStatus() or assign LastAppliedLsn and LastFlushedLsn individually @@ -89,47 +89,44 @@ internal async IAsyncEnumerable Subscribe( } } - private static async IAsyncEnumerable ProcessEnvelope( + private static async IAsyncEnumerable ProcessEnvelope( IEnvelope envelope, - Dictionary registry, + IDictionary> registry, IErrorProcessor errorProcessor - ) where T:class + ) { switch (envelope) { case KoEnvelope error: await errorProcessor.Process(error.Error).ConfigureAwait(false); yield break; - case OkEnvelope okEnvelope: + case OkEnvelope(var value, var messageType): { - var obj = okEnvelope.Value; - var objType = obj.GetType(); - var (messageHandler, methodInfo) = Memoize(registry, objType, MessageHandler); - await ((Task)methodInfo.Invoke(messageHandler, [obj])!).ConfigureAwait(false); - yield return (T)envelope; + var objType = value.GetType(); + var (messageHandler, methodInfo) = Memoize(registry[messageType].Item2, messageType, objType, MessageHandler); + await ((Task)methodInfo.Invoke(messageHandler, [value])!).ConfigureAwait(false); + yield return envelope; yield break; } } } - private static readonly Dictionary Cache = []; - + private static readonly Dictionary Cache = []; private static (IMessageHandler messageHandler, MethodInfo methodInfo) Memoize ( - Dictionary registry, + IMessageHandler messageHandler, + string messageType, Type objType, - Func, Type, (IMessageHandler messageHandler, MethodInfo methodInfo)> func - ) + Func func) { - if (!Cache.TryGetValue(objType, out var entry)) - entry = func(registry, objType); - Cache[objType] = entry; + if (!Cache.TryGetValue(messageType, out var entry)) + entry = func(messageHandler, objType); + Cache[messageType] = entry; return entry; } - private static (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler(Dictionary registry, Type objType) + private static (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler(IMessageHandler messageHandler, Type objType) { - var messageHandler = registry[objType] ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); var methodInfos = messageHandler.GetType().GetMethods(BindingFlags.Instance|BindingFlags.Public); var methodInfo = methodInfos.SingleOrDefault(mi=>mi.GetParameters().Any(pa => pa.ParameterType == objType)) ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); @@ -139,17 +136,19 @@ private static (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHa private static async IAsyncEnumerable ReadExistingRowsFromSnapshot( NpgsqlDataSource dataSource, string snapshotName, - ISubscriptionOptions options, - [EnumeratorCancellation] CancellationToken ct = default + ReplicationDataMapper dataMapper, + TableDescriptorBuilder.MessageTable tableDescriptor, + ISet registeredTypes, + [EnumeratorCancellation] CancellationToken ct = default ) { var connection = await dataSource.OpenConnectionAsync(ct).ConfigureAwait(false); await using var connection1 = connection.ConfigureAwait(false); await foreach (var row in connection.GetRowsFromSnapshot( snapshotName, - options.PublicationOptions.TableDescriptor, - options.DataMapper, - options.PublicationOptions.TypeResolver.Keys().ToHashSet(), + tableDescriptor, + dataMapper, + registeredTypes, ct).ConfigureAwait(false)) yield return row; } diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index 78bc69e..adeed83 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -9,20 +9,22 @@ namespace Blumchen.Subscriptions; public sealed class SubscriptionOptionsBuilder { - private NpgsqlConnectionStringBuilder? _connectionStringBuilder; private NpgsqlDataSource? _dataSource; private PublicationManagement.PublicationSetupOptions _publicationSetupOptions = new(); private ReplicationSlotManagement.ReplicationSlotSetupOptions? _replicationSlotSetupOptions; - private IReplicationDataMapper? _dataMapper; - private readonly Dictionary _registry = []; - + private readonly Dictionary _typeRegistry = []; + private readonly Dictionary> _replicationDataMapperSelector = []; private IErrorProcessor? _errorProcessor; private INamingPolicy? _namingPolicy; - private JsonSerializerContext? _jsonSerializerContext; private readonly TableDescriptorBuilder _tableDescriptorBuilder = new(); private TableDescriptorBuilder.MessageTable? _messageTable; - + private readonly IReplicationJsonBMapper _stringDataMapper = new StringReplicationDataMapper(new StringReplicationDataReader()); + private readonly IReplicationJsonBMapper _objectDataMapper = new ObjectReplicationDataMapper(new ObjectReplicationDataReader()); + private IReplicationJsonBMapper? _jsonDataMapper; + private JsonSerializerContext? _jsonSerializerContext; + + [UsedImplicitly] public SubscriptionOptionsBuilder WithTable( Func builder) @@ -63,7 +65,7 @@ public SubscriptionOptionsBuilder JsonContext(JsonSerializerContext jsonSerializ public SubscriptionOptionsBuilder WithPublicationOptions(PublicationManagement.PublicationSetupOptions publicationOptions) { _publicationSetupOptions = - publicationOptions with { TypeResolver = _publicationSetupOptions.TypeResolver}; + publicationOptions with { RegisteredTypes = _publicationSetupOptions.RegisteredTypes}; return this; } @@ -77,7 +79,26 @@ public SubscriptionOptionsBuilder WithReplicationOptions(ReplicationSlotManageme [UsedImplicitly] public SubscriptionOptionsBuilder Consumes(IMessageHandler handler) where T : class { - _registry.TryAdd(typeof(T), handler); + _typeRegistry.Add(typeof(T), handler); + return this; + } + + [UsedImplicitly] + public SubscriptionOptionsBuilder ConsumesRowObject(IMessageHandler handler) where T : class + => ConsumesRow(handler, RawData.Object); + + [UsedImplicitly] + public SubscriptionOptionsBuilder ConsumesRowString(IMessageHandler handler) where T : class + => ConsumesRow(handler, RawData.String); + + private SubscriptionOptionsBuilder ConsumesRow(IMessageHandler handler, RawData filter) where T : class + { + using var urnEnum = typeof(T) + .GetCustomAttributes(typeof(RawUrnAttribute), false) + .OfType() + .Where(attribute => attribute.Data == filter) + .Select(attribute => attribute.Urn).GetEnumerator(); + while (urnEnum.MoveNext()) _replicationDataMapperSelector.Add(urnEnum.Current.ToString(), new Tuple(_stringDataMapper, handler)); return this; } @@ -93,16 +114,26 @@ internal ISubscriptionOptions Build() _messageTable ??= _tableDescriptorBuilder.Build(); ArgumentNullException.ThrowIfNull(_connectionStringBuilder); ArgumentNullException.ThrowIfNull(_dataSource); - ArgumentNullException.ThrowIfNull(_jsonSerializerContext); + if (_jsonSerializerContext != null) + { - var typeResolver = new JsonTypeResolver(_jsonSerializerContext, _namingPolicy); - foreach (var type in _registry.Keys) typeResolver.WhiteList(type); - _dataMapper = new ReplicationDataMapper(typeResolver); - _publicationSetupOptions = _publicationSetupOptions with { TypeResolver = typeResolver, TableDescriptor = _messageTable}; + var typeResolver = new JsonTypeResolver(_jsonSerializerContext, _namingPolicy); + foreach (var type in _typeRegistry.Keys) + typeResolver.WhiteList(type); + _jsonDataMapper = new JsonReplicationDataMapper(typeResolver, new JsonReplicationDataReader(typeResolver)); + foreach (var (key, value) in typeResolver.RegisteredTypes.Join(_typeRegistry,pair => pair.Value, pair => pair.Key, (pair, valuePair) => (pair.Key, valuePair.Value))) + { + _replicationDataMapperSelector.Add(key,new Tuple(_jsonDataMapper, value)); + } - Ensure(() =>_registry.Keys.Except(_publicationSetupOptions.TypeResolver.Values()), "Unregistered types:{0}"); - Ensure(() => _publicationSetupOptions.TypeResolver.Values().Except(_registry.Keys), "Unregistered consumer for type:{0}"); - if (_registry.Count == 0)_registry.Add(typeof(object), new ObjectTracingConsumer()); + } + + _publicationSetupOptions = _publicationSetupOptions + with + { + RegisteredTypes = _replicationDataMapperSelector.Keys.ToHashSet(), + TableDescriptor = _messageTable + }; return new SubscriptionOptions( _dataSource, @@ -110,13 +141,12 @@ internal ISubscriptionOptions Build() _publicationSetupOptions, _replicationSlotSetupOptions ?? new ReplicationSlotManagement.ReplicationSlotSetupOptions(), _errorProcessor ?? new ConsoleOutErrorProcessor(), - _dataMapper, - _registry); - - static void Ensure(Func> evalFn, string formattedMsg) - { - var misses = evalFn().ToArray(); - if (misses.Length > 0) throw new Exception(string.Format(formattedMsg, string.Join(", ", misses.Select(t => $"'{t.Name}'")))); - } + _replicationDataMapperSelector + ); + //static void Ensure(Func> evalFn, string formattedMsg) + //{ + // var misses = evalFn().ToArray(); + // if (misses.Length > 0) throw new Exception(string.Format(formattedMsg, string.Join(", ", misses.Select(t => $"'{t.Name}'")))); + //} } } diff --git a/src/Subscriber/Contracts.cs b/src/Subscriber/Contracts.cs index 527486b..0dc9c8f 100644 --- a/src/Subscriber/Contracts.cs +++ b/src/Subscriber/Contracts.cs @@ -9,14 +9,14 @@ public record UserCreatedContract( string Name ); - [MessageUrn("user-deleted:v1")] - public record UserDeletedContract( - Guid Id, - string Name - ); + [RawUrn("user-deleted:v1", RawData.Object)] + public interface MessageObjects; + + + [RawUrn("user-modified:v1", RawData.String)] + internal interface MessageString; [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(UserCreatedContract))] - [JsonSerializable(typeof(UserDeletedContract))] internal partial class SourceGenerationContext: JsonSerializerContext; } diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index 2ac77db..e2cbd7e 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -33,9 +33,10 @@ .MessageData("data", new MimeType.Json()) ) .NamingPolicy(new AttributeNamingPolicy()) - .JsonContext(SourceGenerationContext.Default) .Consumes(consumer) - .Consumes(consumer), ct:ct + .JsonContext(SourceGenerationContext.Default) + .ConsumesRowString(consumer) + .ConsumesRowObject(consumer), ct ).GetAsyncEnumerator(ct); await using var cursor1 = cursor.ConfigureAwait(false); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested); @@ -51,9 +52,12 @@ namespace Subscriber { internal class Consumer: IMessageHandler, - IMessageHandler + IMessageHandler, + IMessageHandler { - public Task Handle(UserCreatedContract value) => Console.Out.WriteLineAsync(JsonSerialization.ToJson(value, SourceGenerationContext.Default.UserCreatedContract)); - public Task Handle(UserDeletedContract value) => Console.Out.WriteLineAsync(JsonSerialization.ToJson(value, SourceGenerationContext.Default.UserDeletedContract)); + public Task Handle(string value) => Console.Out.WriteLineAsync(value); + public Task Handle(object value) => Console.Out.WriteLineAsync(value.ToString()); + public Task Handle(UserCreatedContract value) => Console.Out.WriteLineAsync(JsonSerialization.ToJson(value, SourceGenerationContext.Default.UserCreatedContract)); + } } From 0cf7595aa69bbb4c02877e384db7bb0e79a85839 Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 18 Jul 2024 15:37:16 +0200 Subject: [PATCH 32/80] removed unused --- src/Blumchen/Subscriptions/ObjectTracingConsumer.cs | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 src/Blumchen/Subscriptions/ObjectTracingConsumer.cs diff --git a/src/Blumchen/Subscriptions/ObjectTracingConsumer.cs b/src/Blumchen/Subscriptions/ObjectTracingConsumer.cs deleted file mode 100644 index f3d65b8..0000000 --- a/src/Blumchen/Subscriptions/ObjectTracingConsumer.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Blumchen.Subscriptions; - -internal class ObjectTracingConsumer: IMessageHandler -{ - private static ulong _counter = 0; - public Task Handle(object value) - { - Interlocked.Increment(ref _counter); - return Console.Out.WriteLineAsync(); - } -} From 6809b0284d009f1a02334be1d7d59dd800b7747c Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 18 Jul 2024 15:41:13 +0200 Subject: [PATCH 33/80] files reorg --- src/Blumchen/Database/Run.cs | 3 +-- .../DependencyInjection/ServiceCollectionExtensions.cs | 1 + src/Blumchen/DependencyInjection/Worker.cs | 1 + .../{ReplicationMessageHandlers => Replication}/Envelope.cs | 2 +- .../Subscriptions/{ => Replication}/IMessageHandler.cs | 2 +- .../Subscriptions/Replication/IReplicationDataMapper.cs | 1 - .../Subscriptions/Replication/ReplicationDataMapper.cs | 1 - .../{SnapshotReader => Replication}/SnapshotReader.cs | 6 ++---- src/Blumchen/Subscriptions/Subscription.cs | 2 -- src/Subscriber/Program.cs | 1 + src/SubscriberWorker/MessageHandler.cs | 1 + src/SubscriberWorker/Program.cs | 1 + src/Tests/DatabaseFixture.cs | 1 + src/Tests/When_Subscription_Already_Exists.cs | 2 +- .../When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs | 2 +- ...en_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs | 2 +- 16 files changed, 14 insertions(+), 15 deletions(-) rename src/Blumchen/Subscriptions/{ReplicationMessageHandlers => Replication}/Envelope.cs (74%) rename src/Blumchen/Subscriptions/{ => Replication}/IMessageHandler.cs (75%) rename src/Blumchen/Subscriptions/{SnapshotReader => Replication}/SnapshotReader.cs (80%) diff --git a/src/Blumchen/Database/Run.cs b/src/Blumchen/Database/Run.cs index fd1d06a..eca94a5 100644 --- a/src/Blumchen/Database/Run.cs +++ b/src/Blumchen/Database/Run.cs @@ -1,7 +1,6 @@ using System.Data; using System.Runtime.CompilerServices; using Blumchen.Subscriptions.Replication; -using Blumchen.Subscriptions.ReplicationMessageHandlers; using Npgsql; namespace Blumchen.Database; @@ -40,7 +39,7 @@ internal static async IAsyncEnumerable QueryTransactionSnapshot(this string snapshotName, TableDescriptorBuilder.MessageTable tableDescriptor, ISet registeredTypesKeys, - ReplicationDataMapper dataMapper, + IReplicationDataMapper dataMapper, [EnumeratorCancellation] CancellationToken ct) { var transaction = await connection.BeginTransactionAsync(IsolationLevel.RepeatableRead, ct).ConfigureAwait(false); diff --git a/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs index 1cc6d55..5c7ac5c 100644 --- a/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using Blumchen.Subscriptions; +using Blumchen.Subscriptions.Replication; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; diff --git a/src/Blumchen/DependencyInjection/Worker.cs b/src/Blumchen/DependencyInjection/Worker.cs index 4b670ff..119feed 100644 --- a/src/Blumchen/DependencyInjection/Worker.cs +++ b/src/Blumchen/DependencyInjection/Worker.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using Blumchen.Subscriptions; +using Blumchen.Subscriptions.Replication; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; diff --git a/src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs b/src/Blumchen/Subscriptions/Replication/Envelope.cs similarity index 74% rename from src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs rename to src/Blumchen/Subscriptions/Replication/Envelope.cs index 9cd39da..d8e4eae 100644 --- a/src/Blumchen/Subscriptions/ReplicationMessageHandlers/Envelope.cs +++ b/src/Blumchen/Subscriptions/Replication/Envelope.cs @@ -1,4 +1,4 @@ -namespace Blumchen.Subscriptions.ReplicationMessageHandlers; +namespace Blumchen.Subscriptions.Replication; public interface IEnvelope; diff --git a/src/Blumchen/Subscriptions/IMessageHandler.cs b/src/Blumchen/Subscriptions/Replication/IMessageHandler.cs similarity index 75% rename from src/Blumchen/Subscriptions/IMessageHandler.cs rename to src/Blumchen/Subscriptions/Replication/IMessageHandler.cs index 3a55d10..12b4d64 100644 --- a/src/Blumchen/Subscriptions/IMessageHandler.cs +++ b/src/Blumchen/Subscriptions/Replication/IMessageHandler.cs @@ -1,4 +1,4 @@ -namespace Blumchen.Subscriptions; +namespace Blumchen.Subscriptions.Replication; public interface IMessageHandler; diff --git a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs index a6ccd90..327ef0c 100644 --- a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs @@ -1,4 +1,3 @@ -using Blumchen.Subscriptions.ReplicationMessageHandlers; using Npgsql; using Npgsql.Replication.PgOutput.Messages; diff --git a/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs index 52b8c91..b4a2922 100644 --- a/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs @@ -1,7 +1,6 @@ using System.Text.Json; using System.Text.Json.Serialization.Metadata; using Blumchen.Serialization; -using Blumchen.Subscriptions.ReplicationMessageHandlers; using Npgsql; using Npgsql.Replication.PgOutput; using Npgsql.Replication.PgOutput.Messages; diff --git a/src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs b/src/Blumchen/Subscriptions/Replication/SnapshotReader.cs similarity index 80% rename from src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs rename to src/Blumchen/Subscriptions/Replication/SnapshotReader.cs index 4e01ca4..368a95a 100644 --- a/src/Blumchen/Subscriptions/SnapshotReader/SnapshotReader.cs +++ b/src/Blumchen/Subscriptions/Replication/SnapshotReader.cs @@ -1,17 +1,15 @@ using System.Runtime.CompilerServices; using Blumchen.Database; -using Blumchen.Subscriptions.Replication; -using Blumchen.Subscriptions.ReplicationMessageHandlers; using Npgsql; -namespace Blumchen.Subscriptions.SnapshotReader; +namespace Blumchen.Subscriptions.Replication; public static class SnapshotReader { internal static async IAsyncEnumerable GetRowsFromSnapshot(this NpgsqlConnection connection, string snapshotName, TableDescriptorBuilder.MessageTable tableDescriptor, - ReplicationDataMapper dataMapper, + IReplicationDataMapper dataMapper, ISet registeredTypes, [EnumeratorCancellation] CancellationToken ct = default) { diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index bc10dc8..3da1094 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -3,8 +3,6 @@ using Blumchen.Database; using Blumchen.Subscriptions.Management; using Blumchen.Subscriptions.Replication; -using Blumchen.Subscriptions.ReplicationMessageHandlers; -using Blumchen.Subscriptions.SnapshotReader; using Npgsql; using Npgsql.Replication; using Npgsql.Replication.PgOutput; diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index e2cbd7e..6b22160 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -1,5 +1,6 @@ using Blumchen.Serialization; using Blumchen.Subscriptions; +using Blumchen.Subscriptions.Replication; using Commons; using Microsoft.Extensions.Logging; using Npgsql; diff --git a/src/SubscriberWorker/MessageHandler.cs b/src/SubscriberWorker/MessageHandler.cs index 9076b4a..7a34383 100644 --- a/src/SubscriberWorker/MessageHandler.cs +++ b/src/SubscriberWorker/MessageHandler.cs @@ -1,4 +1,5 @@ using Blumchen.Subscriptions; +using Blumchen.Subscriptions.Replication; using Microsoft.Extensions.Logging; #pragma warning disable CS9113 // Parameter is unread. diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index 04f5de1..5a3d68e 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -11,6 +11,7 @@ using SubscriberWorker; using Npgsql; using Blumchen.Subscriptions.Management; +using Blumchen.Subscriptions.Replication; using Polly.Registry; #pragma warning disable CS8601 // Possible null reference assignment. diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index c77e5a6..2887539 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -6,6 +6,7 @@ using Blumchen.Serialization; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Management; +using Blumchen.Subscriptions.Replication; using Npgsql; using Testcontainers.PostgreSql; using Xunit.Abstractions; diff --git a/src/Tests/When_Subscription_Already_Exists.cs b/src/Tests/When_Subscription_Already_Exists.cs index 71b78c9..c6d55e2 100644 --- a/src/Tests/When_Subscription_Already_Exists.cs +++ b/src/Tests/When_Subscription_Already_Exists.cs @@ -2,7 +2,7 @@ using Blumchen.Serialization; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Management; -using Blumchen.Subscriptions.ReplicationMessageHandlers; +using Blumchen.Subscriptions.Replication; using Npgsql; using Xunit.Abstractions; diff --git a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs index 3a52e23..1528a7b 100644 --- a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs +++ b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs @@ -1,7 +1,7 @@ using Blumchen.Publications; using Blumchen.Serialization; using Blumchen.Subscriptions; -using Blumchen.Subscriptions.ReplicationMessageHandlers; +using Blumchen.Subscriptions.Replication; using Npgsql; using Xunit.Abstractions; diff --git a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs index d1029fb..e8c4f3a 100644 --- a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs +++ b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs @@ -1,7 +1,7 @@ using Blumchen.Publications; using Blumchen.Serialization; using Blumchen.Subscriptions; -using Blumchen.Subscriptions.ReplicationMessageHandlers; +using Blumchen.Subscriptions.Replication; using Npgsql; using Xunit.Abstractions; From e20bfa9ed6cf4aaccc162b8845063b9abb27174f Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 18 Jul 2024 15:53:33 +0200 Subject: [PATCH 34/80] marked classes as sealed --- src/Blumchen/Serialization/INamingPolicy.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Blumchen/Serialization/INamingPolicy.cs b/src/Blumchen/Serialization/INamingPolicy.cs index be1be55..f7daa22 100644 --- a/src/Blumchen/Serialization/INamingPolicy.cs +++ b/src/Blumchen/Serialization/INamingPolicy.cs @@ -11,6 +11,6 @@ public abstract record NamingPolicy(Func Bind):INamingPolicy } //This should be used in shared kernel scenario where common library is shared between Pub and Sub -public record FQNNamingPolicy(): NamingPolicy(type => type.FullName!); +public sealed record FQNNamingPolicy(): NamingPolicy(type => type.FullName!); //This policy is better suited for distributed components -public record AttributeNamingPolicy(): NamingPolicy(MessageUrn.ForTypeString); +public sealed record AttributeNamingPolicy(): NamingPolicy(MessageUrn.ForTypeString); From 9656b3bb200ddecdba982ac3e6d47983e293ccf8 Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 18 Jul 2024 15:55:07 +0200 Subject: [PATCH 35/80] Reviewed Atrtibutes class allowed only, not inherit --- .../Serialization/MessageUrnAttribute.cs | 19 ++++++++----------- .../SubscriptionOptionsBuilder.cs | 6 +++--- src/Subscriber/Contracts.cs | 8 ++++---- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/Blumchen/Serialization/MessageUrnAttribute.cs b/src/Blumchen/Serialization/MessageUrnAttribute.cs index f62fe42..ac2b4c5 100644 --- a/src/Blumchen/Serialization/MessageUrnAttribute.cs +++ b/src/Blumchen/Serialization/MessageUrnAttribute.cs @@ -2,7 +2,7 @@ namespace Blumchen.Serialization; -[AttributeUsage(AttributeTargets.Class)] +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] public class MessageUrnAttribute: Attribute { @@ -33,16 +33,15 @@ private static Uri FormatUrn(string urn) } -public enum RawData +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = false)] +public class RawUrnAttribute: Attribute { - String, - Object -} + public enum RawData + { + String, + Object + } -[AttributeUsage(AttributeTargets.Interface, AllowMultiple = true)] -public class RawUrnAttribute: - Attribute -{ public RawData Data { get; } /// @@ -73,8 +72,6 @@ private static Uri FormatUrn(string urn) } } - - public static class MessageUrn { public const string Prefix = "urn:message:"; diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index adeed83..031deea 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -85,13 +85,13 @@ public SubscriptionOptionsBuilder Consumes(IMessageHandler handler) where [UsedImplicitly] public SubscriptionOptionsBuilder ConsumesRowObject(IMessageHandler handler) where T : class - => ConsumesRow(handler, RawData.Object); + => ConsumesRow(handler, RawUrnAttribute.RawData.Object); [UsedImplicitly] public SubscriptionOptionsBuilder ConsumesRowString(IMessageHandler handler) where T : class - => ConsumesRow(handler, RawData.String); + => ConsumesRow(handler, RawUrnAttribute.RawData.String); - private SubscriptionOptionsBuilder ConsumesRow(IMessageHandler handler, RawData filter) where T : class + private SubscriptionOptionsBuilder ConsumesRow(IMessageHandler handler, RawUrnAttribute.RawData filter) where T : class { using var urnEnum = typeof(T) .GetCustomAttributes(typeof(RawUrnAttribute), false) diff --git a/src/Subscriber/Contracts.cs b/src/Subscriber/Contracts.cs index 0dc9c8f..7fb0a57 100644 --- a/src/Subscriber/Contracts.cs +++ b/src/Subscriber/Contracts.cs @@ -9,12 +9,12 @@ public record UserCreatedContract( string Name ); - [RawUrn("user-deleted:v1", RawData.Object)] - public interface MessageObjects; + [RawUrn("user-deleted:v1", RawUrnAttribute.RawData.Object)] + public class MessageObjects; - [RawUrn("user-modified:v1", RawData.String)] - internal interface MessageString; + [RawUrn("user-modified:v1", RawUrnAttribute.RawData.String)] + internal class MessageString; [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(UserCreatedContract))] From 06bd0a11adfb4f57db1085b80700128982d4db72 Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 18 Jul 2024 16:01:42 +0200 Subject: [PATCH 36/80] unused file --- src/Blumchen/Serialization/JsonTypeResolverExtensions.cs | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 src/Blumchen/Serialization/JsonTypeResolverExtensions.cs diff --git a/src/Blumchen/Serialization/JsonTypeResolverExtensions.cs b/src/Blumchen/Serialization/JsonTypeResolverExtensions.cs deleted file mode 100644 index 2d692cd..0000000 --- a/src/Blumchen/Serialization/JsonTypeResolverExtensions.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Blumchen.Serialization; - - -public static class JsonTypeResolverExtensions -{ - internal static IEnumerable Keys(this ITypeResolver? resolver) => resolver?.RegisteredTypes.Keys ?? Enumerable.Empty(); - internal static IEnumerable Values(this ITypeResolver? resolver) => resolver?.RegisteredTypes.Values ?? Enumerable.Empty(); -} From 41a96ded883da1d76ad805258b563722ca3694a3 Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 18 Jul 2024 16:48:38 +0200 Subject: [PATCH 37/80] unused --- src/Blumchen/Serialization/ITypeResolver.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Blumchen/Serialization/ITypeResolver.cs b/src/Blumchen/Serialization/ITypeResolver.cs index 4695d9c..e822c23 100644 --- a/src/Blumchen/Serialization/ITypeResolver.cs +++ b/src/Blumchen/Serialization/ITypeResolver.cs @@ -8,7 +8,6 @@ public interface ITypeResolver { (string, T) Resolve(Type type); Type Resolve(string type); - IDictionary RegisteredTypes { get; } } internal sealed class JsonTypeResolver( From 52b55676e9d440a740ae19f93ac93d3e2676416a Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 18 Jul 2024 16:49:10 +0200 Subject: [PATCH 38/80] expose singleton --- .../Replication/ReplicationDataMapper.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs index b4a2922..82f7f6d 100644 --- a/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs @@ -164,11 +164,19 @@ public async Task ReadFromSnapshot(string typeName, long id, NpgsqlDa internal sealed class ObjectReplicationDataMapper( IReplicationDataReader replicationDataReader -): ReplicationDataMapper(replicationDataReader); +): ReplicationDataMapper(replicationDataReader) +{ + private static readonly Lazy Lazy = new(() => new(new ObjectReplicationDataReader())); + public static ObjectReplicationDataMapper Instance => Lazy.Value; +} internal sealed class StringReplicationDataMapper( IReplicationDataReader replicationDataReader -): ReplicationDataMapper(replicationDataReader); +): ReplicationDataMapper(replicationDataReader) +{ + private static readonly Lazy Lazy = new(() => new(new StringReplicationDataReader())); + public static StringReplicationDataMapper Instance => Lazy.Value; +} internal sealed class JsonReplicationDataMapper( ITypeResolver resolver, From 253d4d4ece9b7e244d2ab2c8780cf28138bc8fb1 Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 19 Jul 2024 18:14:59 +0200 Subject: [PATCH 39/80] reify memoization m --- src/Blumchen/Subscriptions/Memoizer.cs | 38 ++++++++++++ .../Replication/ReplicationDataMapper.cs | 18 +++++- src/Blumchen/Subscriptions/Subscription.cs | 35 +++++------ .../SubscriptionOptionsBuilder.cs | 60 ++++++++++++------- 4 files changed, 106 insertions(+), 45 deletions(-) create mode 100644 src/Blumchen/Subscriptions/Memoizer.cs diff --git a/src/Blumchen/Subscriptions/Memoizer.cs b/src/Blumchen/Subscriptions/Memoizer.cs new file mode 100644 index 0000000..22e27dd --- /dev/null +++ b/src/Blumchen/Subscriptions/Memoizer.cs @@ -0,0 +1,38 @@ +namespace Blumchen.Subscriptions; + +internal class Memoizer(Func func) + where T : notnull +{ + private readonly Dictionary _memoizer = new(); + + private TR Caller(T value1, TU value2) + { + if (_memoizer.TryGetValue(value1, out var result)) + return result; + return _memoizer[value1] = func(value1, value2); + } + + public static Func Execute(Func func) + { + var memoizer = new Memoizer(func); + return (x, y) => memoizer.Caller(x, y); + } +} +internal class Memoizer(Func func) + where T : notnull +{ + private readonly Dictionary _memoizer = new(); + + private TT Caller(T value1, TU value2, TR value3) + { + if (_memoizer.TryGetValue(value1, out var result)) + return result; + return _memoizer[value1] = func(value1, value2, value3); + } + + public static Func Execute(Func func) + { + var memoizer = new Memoizer(func); + return (x, y,z) => memoizer.Caller(x, y,z); + } +} diff --git a/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs index 82f7f6d..e332b84 100644 --- a/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs @@ -63,6 +63,16 @@ public async Task Read(NpgsqlDataReader reader, CancellationToken ct, Ty internal class ReplicationDataMapper(IDictionary> mapperSelector) : IReplicationDataMapper { + private readonly Func>, IReplicationJsonBMapper> _memoizer = Memoizer>, IReplicationJsonBMapper>.Execute(SelectMapper); + + private static IReplicationJsonBMapper SelectMapper(string key, + IDictionary> registry) + => registry.TryGetValue(key, out var tuple) + ? tuple.Item1 + : registry.TryGetValue(SubscriptionOptionsBuilder.WildCard, out tuple) + ? tuple.Item1 + : throw new Exception($"Unexpected message `{key}`"); + public async Task ReadFromReplication(InsertMessage insertMessage, CancellationToken ct) { var id = string.Empty; @@ -86,15 +96,19 @@ public async Task ReadFromReplication(InsertMessage insertMessage, Ca break; } case 2 when column.GetDataTypeName().Equals("jsonb", StringComparison.OrdinalIgnoreCase): - return await mapperSelector[typeName].Item1.ReadFromReplication(id, typeName, column, ct); + + return await _memoizer(typeName, mapperSelector).ReadFromReplication(id, typeName, column, ct); } } - catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or JsonException) + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException + or JsonException) { return new KoEnvelope(ex, id); } + columnNumber++; } + throw new InvalidOperationException("You should not get here"); } diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 3da1094..405b76b 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -16,13 +16,18 @@ namespace Blumchen.Subscriptions; public sealed class Subscription: IAsyncDisposable { + private readonly Func _memoizer = Memoizer.Execute(MessageHandler); + public enum CreateStyle { WhenNotExists, AlwaysRecreate, Never } + private LogicalReplicationConnection? _connection; + private readonly SubscriptionOptionsBuilder _builder = new(); public async IAsyncEnumerable Subscribe( @@ -87,7 +92,7 @@ internal async IAsyncEnumerable Subscribe( } } - private static async IAsyncEnumerable ProcessEnvelope( + private async IAsyncEnumerable ProcessEnvelope( IEnvelope envelope, IDictionary> registry, IErrorProcessor errorProcessor @@ -101,32 +106,20 @@ IErrorProcessor errorProcessor case OkEnvelope(var value, var messageType): { var objType = value.GetType(); - var (messageHandler, methodInfo) = Memoize(registry[messageType].Item2, messageType, objType, MessageHandler); - await ((Task)methodInfo.Invoke(messageHandler, [value])!).ConfigureAwait(false); - yield return envelope; + var (messageHandler, methodInfo) = _memoizer(messageType, registry[messageType].Item2, objType); + await ((Task)methodInfo.Invoke(messageHandler, [value])!).ConfigureAwait(false); + + yield return envelope; yield break; } } } - private static readonly Dictionary Cache = []; - - private static (IMessageHandler messageHandler, MethodInfo methodInfo) Memoize - ( - IMessageHandler messageHandler, - string messageType, - Type objType, - Func func) - { - if (!Cache.TryGetValue(messageType, out var entry)) - entry = func(messageHandler, objType); - Cache[messageType] = entry; - return entry; - } - private static (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler(IMessageHandler messageHandler, Type objType) + private static (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler( + string messageType, IMessageHandler messageHandler, Type objType) { - var methodInfos = messageHandler.GetType().GetMethods(BindingFlags.Instance|BindingFlags.Public); - var methodInfo = methodInfos.SingleOrDefault(mi=>mi.GetParameters().Any(pa => pa.ParameterType == objType)) + var methodInfos = messageHandler.GetType().GetMethods(BindingFlags.Instance | BindingFlags.Public); + var methodInfo = methodInfos.SingleOrDefault(mi => mi.GetParameters().Any(pa => pa.ParameterType == objType)) ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); return (messageHandler, methodInfo); } diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs index 031deea..a5f55e0 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs @@ -9,6 +9,7 @@ namespace Blumchen.Subscriptions; public sealed class SubscriptionOptionsBuilder { + internal const string WildCard = "*"; private NpgsqlConnectionStringBuilder? _connectionStringBuilder; private NpgsqlDataSource? _dataSource; private PublicationManagement.PublicationSetupOptions _publicationSetupOptions = new(); @@ -19,7 +20,7 @@ public sealed class SubscriptionOptionsBuilder private INamingPolicy? _namingPolicy; private readonly TableDescriptorBuilder _tableDescriptorBuilder = new(); private TableDescriptorBuilder.MessageTable? _messageTable; - private readonly IReplicationJsonBMapper _stringDataMapper = new StringReplicationDataMapper(new StringReplicationDataReader()); + private readonly IReplicationJsonBMapper _objectDataMapper = new ObjectReplicationDataMapper(new ObjectReplicationDataReader()); private IReplicationJsonBMapper? _jsonDataMapper; private JsonSerializerContext? _jsonSerializerContext; @@ -85,20 +86,34 @@ public SubscriptionOptionsBuilder Consumes(IMessageHandler handler) where [UsedImplicitly] public SubscriptionOptionsBuilder ConsumesRowObject(IMessageHandler handler) where T : class - => ConsumesRow(handler, RawUrnAttribute.RawData.Object); + => ConsumesRow(handler, RawUrnAttribute.RawData.Object, ObjectReplicationDataMapper.Instance); [UsedImplicitly] public SubscriptionOptionsBuilder ConsumesRowString(IMessageHandler handler) where T : class - => ConsumesRow(handler, RawUrnAttribute.RawData.String); + => ConsumesRow(handler, RawUrnAttribute.RawData.String, StringReplicationDataMapper.Instance); + + [UsedImplicitly] + public SubscriptionOptionsBuilder ConsumesRowStrings(IMessageHandler handler) + { + _replicationDataMapperSelector.Add(WildCard, new Tuple(StringReplicationDataMapper.Instance, handler)); + return this; + } - private SubscriptionOptionsBuilder ConsumesRow(IMessageHandler handler, RawUrnAttribute.RawData filter) where T : class + [UsedImplicitly] + public SubscriptionOptionsBuilder ConsumesRowObjects(IMessageHandler handler) + { + _replicationDataMapperSelector.Add(WildCard, new Tuple(ObjectReplicationDataMapper.Instance, handler)); + return this; + } + + private SubscriptionOptionsBuilder ConsumesRow(IMessageHandler handler, RawUrnAttribute.RawData filter, IReplicationJsonBMapper dataMapper) where T : class { using var urnEnum = typeof(T) .GetCustomAttributes(typeof(RawUrnAttribute), false) .OfType() .Where(attribute => attribute.Data == filter) .Select(attribute => attribute.Urn).GetEnumerator(); - while (urnEnum.MoveNext()) _replicationDataMapperSelector.Add(urnEnum.Current.ToString(), new Tuple(_stringDataMapper, handler)); + while (urnEnum.MoveNext()) _replicationDataMapperSelector.Add(urnEnum.Current.ToString(), new Tuple(dataMapper, handler)); return this; } @@ -114,27 +129,31 @@ internal ISubscriptionOptions Build() _messageTable ??= _tableDescriptorBuilder.Build(); ArgumentNullException.ThrowIfNull(_connectionStringBuilder); ArgumentNullException.ThrowIfNull(_dataSource); - if (_jsonSerializerContext != null) + + if(_typeRegistry.Count > 0) { - - var typeResolver = new JsonTypeResolver(_jsonSerializerContext, _namingPolicy); - foreach (var type in _typeRegistry.Keys) - typeResolver.WhiteList(type); - _jsonDataMapper = new JsonReplicationDataMapper(typeResolver, new JsonReplicationDataReader(typeResolver)); - foreach (var (key, value) in typeResolver.RegisteredTypes.Join(_typeRegistry,pair => pair.Value, pair => pair.Key, (pair, valuePair) => (pair.Key, valuePair.Value))) + if (_jsonSerializerContext != null) { - _replicationDataMapperSelector.Add(key,new Tuple(_jsonDataMapper, value)); + var typeResolver = new JsonTypeResolver(_jsonSerializerContext, _namingPolicy); + foreach (var type in _typeRegistry.Keys) + typeResolver.WhiteList(type); + + _jsonDataMapper = new JsonReplicationDataMapper(typeResolver, new JsonReplicationDataReader(typeResolver)); + + foreach (var (key, value) in typeResolver.RegisteredTypes.Join(_typeRegistry,pair => pair.Value, pair => pair.Key, (pair, valuePair) => (pair.Key, valuePair.Value))) + _replicationDataMapperSelector.Add(key,new Tuple(_jsonDataMapper, value)); + } + else + { + throw new ConfigurationException($"`${nameof(Consumes)}<>` requires a valid `{nameof(JsonContext)}`."); } - } - _publicationSetupOptions = _publicationSetupOptions with { - RegisteredTypes = _replicationDataMapperSelector.Keys.ToHashSet(), + RegisteredTypes = _replicationDataMapperSelector.Keys.Except(new [] { WildCard }).ToHashSet(), TableDescriptor = _messageTable }; - return new SubscriptionOptions( _dataSource, _connectionStringBuilder, @@ -143,10 +162,7 @@ internal ISubscriptionOptions Build() _errorProcessor ?? new ConsoleOutErrorProcessor(), _replicationDataMapperSelector ); - //static void Ensure(Func> evalFn, string formattedMsg) - //{ - // var misses = evalFn().ToArray(); - // if (misses.Length > 0) throw new Exception(string.Format(formattedMsg, string.Join(", ", misses.Select(t => $"'{t.Name}'")))); - //} } } + +public class ConfigurationException(string message): Exception(message); From 678b2c81819eb9acf5ba5dba7887b7115dd96ebc Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 19 Jul 2024 18:56:19 +0200 Subject: [PATCH 40/80] formatting stuff --- src/Blumchen/Subscriptions/Subscription.cs | 75 ++++++++++++---------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 405b76b..60ce0e4 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -7,17 +7,27 @@ using Npgsql.Replication; using Npgsql.Replication.PgOutput; using Npgsql.Replication.PgOutput.Messages; +using static Blumchen.Subscriptions.Management.ReplicationSlotManagement.CreateReplicationSlotResult; namespace Blumchen.Subscriptions; -using static PublicationManagement; -using static ReplicationSlotManagement; -using static ReplicationSlotManagement.CreateReplicationSlotResult; - public sealed class Subscription: IAsyncDisposable { - private readonly Func _memoizer = Memoizer.Execute(MessageHandler); + private LogicalReplicationConnection? _connection; + private readonly SubscriptionOptionsBuilder _builder = new(); + + private readonly Func + _messageHandler = Memoizer.Execute(MessageHandler); + + private static (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler( + string messageType, IMessageHandler messageHandler, Type objType) + { + var methodInfos = messageHandler.GetType().GetMethods(BindingFlags.Instance | BindingFlags.Public); + var methodInfo = methodInfos.SingleOrDefault(mi => mi.GetParameters().Any(pa => pa.ParameterType == objType)) + ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); + return (messageHandler, methodInfo); + } public enum CreateStyle { @@ -26,10 +36,6 @@ public enum CreateStyle Never } - private LogicalReplicationConnection? _connection; - - private readonly SubscriptionOptionsBuilder _builder = new(); - public async IAsyncEnumerable Subscribe( Func builder, [EnumeratorCancellation] CancellationToken ct = default @@ -40,18 +46,20 @@ public async IAsyncEnumerable Subscribe( } internal async IAsyncEnumerable Subscribe( - ISubscriptionOptions subscriptionOptions, - [EnumeratorCancellation] CancellationToken ct = default - ) + ISubscriptionOptions subscriptionOptions, + [EnumeratorCancellation] CancellationToken ct = default + ) { - var (dataSource, connectionStringBuilder, publicationSetupOptions, replicationSlotSetupOptions, errorProcessor, registry) = subscriptionOptions; + var (dataSource, connectionStringBuilder, publicationSetupOptions, replicationSlotSetupOptions, errorProcessor, + registry) = subscriptionOptions; await dataSource.EnsureTableExists(publicationSetupOptions.TableDescriptor, ct).ConfigureAwait(false); _connection = new LogicalReplicationConnection(connectionStringBuilder.ConnectionString); await _connection.Open(ct).ConfigureAwait(false); await dataSource.SetupPublication(publicationSetupOptions, ct).ConfigureAwait(false); - var result = await dataSource.SetupReplicationSlot(_connection, replicationSlotSetupOptions, ct).ConfigureAwait(false); + var result = await dataSource.SetupReplicationSlot(_connection, replicationSlotSetupOptions, ct) + .ConfigureAwait(false); var replicationDataMapper = new ReplicationDataMapper(registry); PgOutputReplicationSlot slot; @@ -68,23 +76,29 @@ internal async IAsyncEnumerable Subscribe( ) ); - await foreach (var envelope in ReadExistingRowsFromSnapshot(dataSource, created.SnapshotName, replicationDataMapper, publicationSetupOptions.TableDescriptor, publicationSetupOptions.RegisteredTypes, ct).ConfigureAwait(false)) + await foreach (var envelope in ReadExistingRowsFromSnapshot(dataSource, created.SnapshotName, + replicationDataMapper, publicationSetupOptions.TableDescriptor, + publicationSetupOptions.RegisteredTypes, ct).ConfigureAwait(false)) { - await foreach (var subscribe in ProcessEnvelope(envelope, registry, errorProcessor).WithCancellation(ct).ConfigureAwait(false)) + await foreach (var subscribe in ProcessEnvelope(envelope, registry, errorProcessor).WithCancellation(ct) + .ConfigureAwait(false)) yield return subscribe; } } await foreach (var message in _connection.StartReplication(slot, - new PgOutputReplicationOptions(publicationSetupOptions.PublicationName, 1, replicationSlotSetupOptions.Binary), ct).ConfigureAwait(false)) + new PgOutputReplicationOptions(publicationSetupOptions.PublicationName, 1, + replicationSlotSetupOptions.Binary), ct).ConfigureAwait(false)) { if (message is InsertMessage insertMessage) { var envelope = await replicationDataMapper.ReadFromReplication(insertMessage, ct).ConfigureAwait(false); - await foreach (var subscribe in ProcessEnvelope(envelope, registry, errorProcessor).WithCancellation(ct).ConfigureAwait(false)) + await foreach (var subscribe in ProcessEnvelope(envelope, registry, errorProcessor).WithCancellation(ct) + .ConfigureAwait(false)) yield return subscribe; } + // Always call SetReplicationStatus() or assign LastAppliedLsn and LastFlushedLsn individually // so that Npgsql can inform the server which WAL files can be removed/recycled. _connection.SetReplicationStatus(message.WalEnd); @@ -104,33 +118,24 @@ IErrorProcessor errorProcessor await errorProcessor.Process(error.Error).ConfigureAwait(false); yield break; case OkEnvelope(var value, var messageType): - { - var objType = value.GetType(); - var (messageHandler, methodInfo) = _memoizer(messageType, registry[messageType].Item2, objType); + { + var (messageHandler, methodInfo) = + _messageHandler(messageType, registry[messageType].Item2, value.GetType()); await ((Task)methodInfo.Invoke(messageHandler, [value])!).ConfigureAwait(false); yield return envelope; - yield break; - } + yield break; + } } } - private static (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler( - string messageType, IMessageHandler messageHandler, Type objType) - { - var methodInfos = messageHandler.GetType().GetMethods(BindingFlags.Instance | BindingFlags.Public); - var methodInfo = methodInfos.SingleOrDefault(mi => mi.GetParameters().Any(pa => pa.ParameterType == objType)) - ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); - return (messageHandler, methodInfo); - } - private static async IAsyncEnumerable ReadExistingRowsFromSnapshot( NpgsqlDataSource dataSource, string snapshotName, ReplicationDataMapper dataMapper, TableDescriptorBuilder.MessageTable tableDescriptor, ISet registeredTypes, - [EnumeratorCancellation] CancellationToken ct = default + [EnumeratorCancellation] CancellationToken ct = default ) { var connection = await dataSource.OpenConnectionAsync(ct).ConfigureAwait(false); @@ -146,7 +151,7 @@ private static async IAsyncEnumerable ReadExistingRowsFromSnapshot( public async ValueTask DisposeAsync() { - if(_connection != null) + if (_connection != null) await _connection.DisposeAsync().ConfigureAwait(false); } } From c60bc895b2ae999b458fd17e7853363d11f6cbb8 Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 19 Jul 2024 18:59:00 +0200 Subject: [PATCH 41/80] class renamed to PublisherOptions and SubscriberOptions. Move classes to own files --- .../ServiceCollectionExtensions.cs | 1 - src/Blumchen/DependencyInjection/Worker.cs | 2 +- .../WorkerOptionsBuilder.cs | 12 +- .../MessageAppender.cs | 2 +- .../OptionsBuilder.cs} | 11 +- .../PublisherOptions.cs | 2 +- .../ISubscriberOptions.cs} | 9 +- .../OptionsBuilder.cs} | 39 ++-- .../Management/PublicationManagement.cs | 2 - .../Replication/IReplicationDataMapper.cs | 73 ++++++- .../Replication/IReplicationDataReader.cs | 56 +++++ .../Replication/IReplicationJsonBMapper.cs | 74 +++++++ .../Replication/ReplicationDataMapper.cs | 198 ------------------ src/Blumchen/Subscriptions/Subscription.cs | 13 +- ...leOptions.cs => TableDescriptorBuilder.cs} | 0 src/Publisher/Program.cs | 4 +- src/SubscriberWorker/MessageHandler.cs | 1 - src/Tests/DatabaseFixture.cs | 5 +- src/Tests/When_Subscription_Already_Exists.cs | 4 +- ...ption_Does_Not_Exist_And_Table_Is_Empty.cs | 4 +- ...n_Does_Not_Exist_And_Table_Is_Not_Empty.cs | 4 +- 21 files changed, 259 insertions(+), 257 deletions(-) rename src/Blumchen/{Publications => Publisher}/MessageAppender.cs (99%) rename src/Blumchen/{Publications/PublisherSetupOptionsBuilder.cs => Publisher/OptionsBuilder.cs} (78%) rename src/Blumchen/{Publications => Publisher}/PublisherOptions.cs (96%) rename src/Blumchen/{Subscriptions/ISubscriptionOptions.cs => Subscriber/ISubscriberOptions.cs} (88%) rename src/Blumchen/{Subscriptions/SubscriptionOptionsBuilder.cs => Subscriber/OptionsBuilder.cs} (77%) create mode 100644 src/Blumchen/Subscriptions/Replication/IReplicationDataReader.cs create mode 100644 src/Blumchen/Subscriptions/Replication/IReplicationJsonBMapper.cs delete mode 100644 src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs rename src/Blumchen/{MessageTableOptions.cs => TableDescriptorBuilder.cs} (100%) diff --git a/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs index 5c7ac5c..b1a2081 100644 --- a/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs @@ -1,4 +1,3 @@ -using Blumchen.Subscriptions; using Blumchen.Subscriptions.Replication; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; diff --git a/src/Blumchen/DependencyInjection/Worker.cs b/src/Blumchen/DependencyInjection/Worker.cs index 119feed..e828089 100644 --- a/src/Blumchen/DependencyInjection/Worker.cs +++ b/src/Blumchen/DependencyInjection/Worker.cs @@ -29,7 +29,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) await options.ResiliencePipeline.ExecuteAsync(async token => { await using var subscription = new Subscription(); - await using var cursor = subscription.Subscribe(options.SubscriptionOptions, ct: token) + await using var cursor = subscription.Subscribe(options.SubscriberOptions, ct: token) .GetAsyncEnumerator(token); Notify(logger, LogLevel.Information,"{WorkerName} started", WorkerName); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !token.IsCancellationRequested) diff --git a/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs b/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs index 07c79c5..750878a 100644 --- a/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs +++ b/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs @@ -1,27 +1,27 @@ -using Blumchen.Subscriptions; +using Blumchen.Subscriber; using Polly; namespace Blumchen.DependencyInjection; -public record WorkerOptions(ResiliencePipeline ResiliencePipeline, ISubscriptionOptions SubscriptionOptions); +public record WorkerOptions(ResiliencePipeline ResiliencePipeline, ISubscriberOptions SubscriberOptions); public interface IWorkerOptionsBuilder { IWorkerOptionsBuilder ResiliencyPipeline(ResiliencePipeline resiliencePipeline); - IWorkerOptionsBuilder Subscription(Func? builder); + IWorkerOptionsBuilder Subscription(Func? builder); WorkerOptions Build(); } internal sealed class WorkerOptionsBuilder: IWorkerOptionsBuilder { private ResiliencePipeline? _resiliencePipeline = default; - private Func? _builder; + private Func? _builder; public IWorkerOptionsBuilder ResiliencyPipeline(ResiliencePipeline resiliencePipeline) { _resiliencePipeline = resiliencePipeline; return this; - }public IWorkerOptionsBuilder Subscription(Func? builder) + }public IWorkerOptionsBuilder Subscription(Func? builder) { _builder = builder; return this; @@ -31,7 +31,7 @@ public WorkerOptions Build() { ArgumentNullException.ThrowIfNull(_resiliencePipeline); ArgumentNullException.ThrowIfNull(_builder); - return new(_resiliencePipeline, _builder(new SubscriptionOptionsBuilder()).Build()); + return new(_resiliencePipeline, _builder(new OptionsBuilder()).Build()); } } diff --git a/src/Blumchen/Publications/MessageAppender.cs b/src/Blumchen/Publisher/MessageAppender.cs similarity index 99% rename from src/Blumchen/Publications/MessageAppender.cs rename to src/Blumchen/Publisher/MessageAppender.cs index 5fd963c..da7d43c 100644 --- a/src/Blumchen/Publications/MessageAppender.cs +++ b/src/Blumchen/Publisher/MessageAppender.cs @@ -3,7 +3,7 @@ using Blumchen.Serialization; using Npgsql; -namespace Blumchen.Publications; +namespace Blumchen.Publisher; public static class MessageAppender { diff --git a/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs b/src/Blumchen/Publisher/OptionsBuilder.cs similarity index 78% rename from src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs rename to src/Blumchen/Publisher/OptionsBuilder.cs index 00be8f1..eb7785b 100644 --- a/src/Blumchen/Publications/PublisherSetupOptionsBuilder.cs +++ b/src/Blumchen/Publisher/OptionsBuilder.cs @@ -1,12 +1,11 @@ using System.Text.Json.Serialization; -using System.Text.Json.Serialization.Metadata; using Blumchen.Serialization; using JetBrains.Annotations; using static Blumchen.TableDescriptorBuilder; -namespace Blumchen.Publications; +namespace Blumchen.Publisher; -public class PublisherSetupOptionsBuilder +public class OptionsBuilder { private INamingPolicy? _namingPolicy; private JsonSerializerContext? _jsonSerializerContext; @@ -14,21 +13,21 @@ public class PublisherSetupOptionsBuilder private MessageTable? _tableDescriptor; [UsedImplicitly] - public PublisherSetupOptionsBuilder NamingPolicy(INamingPolicy namingPolicy) + public OptionsBuilder NamingPolicy(INamingPolicy namingPolicy) { _namingPolicy = namingPolicy; return this; } [UsedImplicitly] - public PublisherSetupOptionsBuilder JsonContext(JsonSerializerContext jsonSerializerContext) + public OptionsBuilder JsonContext(JsonSerializerContext jsonSerializerContext) { _jsonSerializerContext = jsonSerializerContext; return this; } [UsedImplicitly] - public PublisherSetupOptionsBuilder WithTable(Func builder) + public OptionsBuilder WithTable(Func builder) { _tableDescriptor = builder(TableDescriptorBuilder).Build(); return this; diff --git a/src/Blumchen/Publications/PublisherOptions.cs b/src/Blumchen/Publisher/PublisherOptions.cs similarity index 96% rename from src/Blumchen/Publications/PublisherOptions.cs rename to src/Blumchen/Publisher/PublisherOptions.cs index 9c16d3f..48ace93 100644 --- a/src/Blumchen/Publications/PublisherOptions.cs +++ b/src/Blumchen/Publisher/PublisherOptions.cs @@ -3,7 +3,7 @@ using Blumchen.Serialization; using Npgsql; -namespace Blumchen.Publications; +namespace Blumchen.Publisher; public record PublisherOptions(TableDescriptorBuilder.MessageTable TableDescriptor, ITypeResolver JsonTypeResolver); diff --git a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs b/src/Blumchen/Subscriber/ISubscriberOptions.cs similarity index 88% rename from src/Blumchen/Subscriptions/ISubscriptionOptions.cs rename to src/Blumchen/Subscriber/ISubscriberOptions.cs index 74aca0e..caccd5a 100644 --- a/src/Blumchen/Subscriptions/ISubscriptionOptions.cs +++ b/src/Blumchen/Subscriber/ISubscriberOptions.cs @@ -1,12 +1,13 @@ +using Blumchen.Subscriptions; using Blumchen.Subscriptions.Replication; using JetBrains.Annotations; using Npgsql; using static Blumchen.Subscriptions.Management.PublicationManagement; using static Blumchen.Subscriptions.Management.ReplicationSlotManagement; -namespace Blumchen.Subscriptions; +namespace Blumchen.Subscriber; -public interface ISubscriptionOptions +public interface ISubscriberOptions { [UsedImplicitly] NpgsqlDataSource DataSource { get; } [UsedImplicitly] NpgsqlConnectionStringBuilder ConnectionStringBuilder { get; } @@ -24,10 +25,10 @@ void Deconstruct( out IDictionary> registry); } -internal record SubscriptionOptions( +internal record SubscriberOptions( NpgsqlDataSource DataSource, NpgsqlConnectionStringBuilder ConnectionStringBuilder, PublicationSetupOptions PublicationOptions, ReplicationSlotSetupOptions ReplicationOptions, IErrorProcessor ErrorProcessor, - IDictionary> Registry): ISubscriptionOptions; + IDictionary> Registry): ISubscriberOptions; diff --git a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs similarity index 77% rename from src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs rename to src/Blumchen/Subscriber/OptionsBuilder.cs index a5f55e0..ae86d0b 100644 --- a/src/Blumchen/Subscriptions/SubscriptionOptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -1,13 +1,14 @@ +using System.Text.Json.Serialization; using Blumchen.Serialization; +using Blumchen.Subscriptions; using Blumchen.Subscriptions.Management; using Blumchen.Subscriptions.Replication; using JetBrains.Annotations; using Npgsql; -using System.Text.Json.Serialization; -namespace Blumchen.Subscriptions; +namespace Blumchen.Subscriber; -public sealed class SubscriptionOptionsBuilder +public sealed class OptionsBuilder { internal const string WildCard = "*"; private NpgsqlConnectionStringBuilder? _connectionStringBuilder; @@ -27,7 +28,7 @@ public sealed class SubscriptionOptionsBuilder [UsedImplicitly] - public SubscriptionOptionsBuilder WithTable( + public OptionsBuilder WithTable( Func builder) { _messageTable = builder(_tableDescriptorBuilder).Build(); @@ -35,35 +36,35 @@ public SubscriptionOptionsBuilder WithTable( } [UsedImplicitly] - public SubscriptionOptionsBuilder ConnectionString(string connectionString) + public OptionsBuilder ConnectionString(string connectionString) { _connectionStringBuilder = new NpgsqlConnectionStringBuilder(connectionString); return this; } [UsedImplicitly] - public SubscriptionOptionsBuilder DataSource(NpgsqlDataSource dataSource) + public OptionsBuilder DataSource(NpgsqlDataSource dataSource) { _dataSource = dataSource; return this; } [UsedImplicitly] - public SubscriptionOptionsBuilder NamingPolicy(INamingPolicy namingPolicy) + public OptionsBuilder NamingPolicy(INamingPolicy namingPolicy) { _namingPolicy = namingPolicy; return this; } [UsedImplicitly] - public SubscriptionOptionsBuilder JsonContext(JsonSerializerContext jsonSerializerContext) + public OptionsBuilder JsonContext(JsonSerializerContext jsonSerializerContext) { _jsonSerializerContext = jsonSerializerContext; return this; } [UsedImplicitly] - public SubscriptionOptionsBuilder WithPublicationOptions(PublicationManagement.PublicationSetupOptions publicationOptions) + public OptionsBuilder WithPublicationOptions(PublicationManagement.PublicationSetupOptions publicationOptions) { _publicationSetupOptions = publicationOptions with { RegisteredTypes = _publicationSetupOptions.RegisteredTypes}; @@ -71,42 +72,42 @@ public SubscriptionOptionsBuilder WithPublicationOptions(PublicationManagement.P } [UsedImplicitly] - public SubscriptionOptionsBuilder WithReplicationOptions(ReplicationSlotManagement.ReplicationSlotSetupOptions replicationSlotOptions) + public OptionsBuilder WithReplicationOptions(ReplicationSlotManagement.ReplicationSlotSetupOptions replicationSlotOptions) { _replicationSlotSetupOptions = replicationSlotOptions; return this; } [UsedImplicitly] - public SubscriptionOptionsBuilder Consumes(IMessageHandler handler) where T : class + public OptionsBuilder Consumes(IMessageHandler handler) where T : class { _typeRegistry.Add(typeof(T), handler); return this; } [UsedImplicitly] - public SubscriptionOptionsBuilder ConsumesRowObject(IMessageHandler handler) where T : class + public OptionsBuilder ConsumesRowObject(IMessageHandler handler) where T : class => ConsumesRow(handler, RawUrnAttribute.RawData.Object, ObjectReplicationDataMapper.Instance); [UsedImplicitly] - public SubscriptionOptionsBuilder ConsumesRowString(IMessageHandler handler) where T : class + public OptionsBuilder ConsumesRowString(IMessageHandler handler) where T : class => ConsumesRow(handler, RawUrnAttribute.RawData.String, StringReplicationDataMapper.Instance); [UsedImplicitly] - public SubscriptionOptionsBuilder ConsumesRowStrings(IMessageHandler handler) + public OptionsBuilder ConsumesRowStrings(IMessageHandler handler) { _replicationDataMapperSelector.Add(WildCard, new Tuple(StringReplicationDataMapper.Instance, handler)); return this; } [UsedImplicitly] - public SubscriptionOptionsBuilder ConsumesRowObjects(IMessageHandler handler) + public OptionsBuilder ConsumesRowObjects(IMessageHandler handler) { _replicationDataMapperSelector.Add(WildCard, new Tuple(ObjectReplicationDataMapper.Instance, handler)); return this; } - private SubscriptionOptionsBuilder ConsumesRow(IMessageHandler handler, RawUrnAttribute.RawData filter, IReplicationJsonBMapper dataMapper) where T : class + private OptionsBuilder ConsumesRow(IMessageHandler handler, RawUrnAttribute.RawData filter, IReplicationJsonBMapper dataMapper) where T : class { using var urnEnum = typeof(T) .GetCustomAttributes(typeof(RawUrnAttribute), false) @@ -118,13 +119,13 @@ private SubscriptionOptionsBuilder ConsumesRow(IMessageHandler handle } [UsedImplicitly] - public SubscriptionOptionsBuilder WithErrorProcessor(IErrorProcessor? errorProcessor) + public OptionsBuilder WithErrorProcessor(IErrorProcessor? errorProcessor) { _errorProcessor = errorProcessor; return this; } - internal ISubscriptionOptions Build() + internal ISubscriberOptions Build() { _messageTable ??= _tableDescriptorBuilder.Build(); ArgumentNullException.ThrowIfNull(_connectionStringBuilder); @@ -154,7 +155,7 @@ internal ISubscriptionOptions Build() RegisteredTypes = _replicationDataMapperSelector.Keys.Except(new [] { WildCard }).ToHashSet(), TableDescriptor = _messageTable }; - return new SubscriptionOptions( + return new SubscriberOptions( _dataSource, _connectionStringBuilder, _publicationSetupOptions, diff --git a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs index baf1bde..0a9ff27 100644 --- a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs +++ b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs @@ -1,6 +1,4 @@ -using System.Text.Json.Serialization.Metadata; using Blumchen.Database; -using Blumchen.Serialization; using Npgsql; #pragma warning disable CA2208 diff --git a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs index 327ef0c..d248e01 100644 --- a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs @@ -1,5 +1,8 @@ using Npgsql; +using Npgsql.Replication.PgOutput; using Npgsql.Replication.PgOutput.Messages; +using System.Text.Json; +using Blumchen.Subscriber; namespace Blumchen.Subscriptions.Replication; @@ -8,6 +11,74 @@ public interface IReplicationDataMapper Task ReadFromSnapshot(NpgsqlDataReader reader, CancellationToken ct); Task ReadFromReplication(InsertMessage insertMessage, CancellationToken ct); +} + +internal class ReplicationDataMapper(IDictionary> mapperSelector) + : IReplicationDataMapper +{ + private readonly Func>, IReplicationJsonBMapper> _memoizer = Memoizer>, IReplicationJsonBMapper>.Execute(SelectMapper); + + private static IReplicationJsonBMapper SelectMapper(string key, + IDictionary> registry) + => registry.TryGetValue(key, out var tuple) + ? tuple.Item1 + : registry.TryGetValue(OptionsBuilder.WildCard, out tuple) + ? tuple.Item1 + : throw new Exception($"Unexpected message `{key}`"); + + public async Task ReadFromReplication(InsertMessage insertMessage, CancellationToken ct) + { + var id = string.Empty; + var columnNumber = 0; + var typeName = string.Empty; + await foreach (var column in insertMessage.NewRow.ConfigureAwait(false)) + { + try + { + switch (columnNumber) + { + case 0: + id = column.Kind == TupleDataKind.BinaryValue + ? (await column.Get(ct).ConfigureAwait(false)).ToString() + : await column.Get(ct).ConfigureAwait(false); + break; + case 1: + using (var textReader = column.GetTextReader()) + { + typeName = await textReader.ReadToEndAsync(ct).ConfigureAwait(false); + break; + } + case 2 when column.GetDataTypeName().Equals("jsonb", StringComparison.OrdinalIgnoreCase): - + return await _memoizer(typeName, mapperSelector).ReadFromReplication(id, typeName, column, ct); + } + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException + or JsonException) + { + return new KoEnvelope(ex, id); + } + + columnNumber++; + } + + throw new InvalidOperationException("You should not get here"); + } + + public async Task ReadFromSnapshot(NpgsqlDataReader reader, CancellationToken ct) + { + long id = default; + try + { + id = reader.GetInt64(0); + var typeName = reader.GetString(1); + + return await mapperSelector[typeName].Item1.ReadFromSnapshot(typeName, id, reader, ct); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or JsonException) + { + return new KoEnvelope(ex, id.ToString()); + } + } } + diff --git a/src/Blumchen/Subscriptions/Replication/IReplicationDataReader.cs b/src/Blumchen/Subscriptions/Replication/IReplicationDataReader.cs new file mode 100644 index 0000000..3e86ccf --- /dev/null +++ b/src/Blumchen/Subscriptions/Replication/IReplicationDataReader.cs @@ -0,0 +1,56 @@ +using Blumchen.Serialization; +using Npgsql; +using Npgsql.Replication.PgOutput; + +namespace Blumchen.Subscriptions.Replication; + +internal interface IReplicationDataReader +{ + Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default); + Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default); +} +internal class ObjectReplicationDataReader: IReplicationDataReader +{ + public Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default) + => replicationValue.Get(ct).AsTask(); + + + public Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default) + => reader.GetFieldValueAsync(2, ct); +} + +internal class StringReplicationDataReader: IReplicationDataReader +{ + public async Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default) + { + using var tr = replicationValue.GetTextReader(); + return await tr.ReadToEndAsync(ct).ConfigureAwait(false); + } + + + public async Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default) + { + using var tr = await reader.GetTextReaderAsync(2, ct).ConfigureAwait(false); + return await tr.ReadToEndAsync(ct).ConfigureAwait(false); + } +} + +internal class JsonReplicationDataReader(JsonTypeResolver resolver): IReplicationDataReader +{ + public async Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default) + { + ArgumentNullException.ThrowIfNull(type); + await using var stream = replicationValue.GetStream(); + return await JsonSerialization.FromJsonAsync(type, stream, resolver.SerializationContext, ct) + .ConfigureAwait(false); + } + + public async Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default) + { + ArgumentNullException.ThrowIfNull(type); + var stream = await reader.GetStreamAsync(2, ct).ConfigureAwait(false); + await using var stream1 = stream.ConfigureAwait(false); + return await JsonSerialization.FromJsonAsync(type, stream, resolver.SerializationContext, ct) + .ConfigureAwait(false); + } +} diff --git a/src/Blumchen/Subscriptions/Replication/IReplicationJsonBMapper.cs b/src/Blumchen/Subscriptions/Replication/IReplicationJsonBMapper.cs new file mode 100644 index 0000000..89deb4c --- /dev/null +++ b/src/Blumchen/Subscriptions/Replication/IReplicationJsonBMapper.cs @@ -0,0 +1,74 @@ +using Blumchen.Serialization; +using Npgsql; +using Npgsql.Replication.PgOutput; +using System.Text.Json.Serialization.Metadata; +using System.Text.Json; + +namespace Blumchen.Subscriptions.Replication; + +public interface IReplicationJsonBMapper +{ + Task ReadFromReplication(string id, string typeName, ReplicationValue column, + CancellationToken ct); + + Task ReadFromSnapshot(string typeName, long id, NpgsqlDataReader reader, CancellationToken ct); +} +internal class ReplicationDataMapper( + IReplicationDataReader replicationDataReader + , ITypeResolver? resolver = default + ): IReplicationJsonBMapper +{ + public async Task ReadFromReplication(string id, string typeName, ReplicationValue column, + CancellationToken ct) + { + + try + { + var type = resolver?.Resolve(typeName); + var value = await replicationDataReader.Read(column, ct, type).ConfigureAwait(false) ?? + throw new ArgumentNullException(); + return new OkEnvelope(value, typeName); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException + or JsonException) + { + return new KoEnvelope(ex, id); + } + } + + public async Task ReadFromSnapshot(string typeName, long id, NpgsqlDataReader reader, CancellationToken ct) + { + try + { + var eventType = resolver?.Resolve(typeName); + ArgumentNullException.ThrowIfNull(eventType, typeName); + var value = await replicationDataReader.Read(reader, ct, eventType).ConfigureAwait(false) ?? throw new ArgumentNullException(); + return new OkEnvelope(value, typeName); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or JsonException) + { + return new KoEnvelope(ex, id.ToString()); + } + } +} + +internal sealed class ObjectReplicationDataMapper( + IReplicationDataReader replicationDataReader +): ReplicationDataMapper(replicationDataReader) +{ + private static readonly Lazy Lazy = new(() => new(new ObjectReplicationDataReader())); + public static ObjectReplicationDataMapper Instance => Lazy.Value; +} + +internal sealed class StringReplicationDataMapper( + IReplicationDataReader replicationDataReader +): ReplicationDataMapper(replicationDataReader) +{ + private static readonly Lazy Lazy = new(() => new(new StringReplicationDataReader())); + public static StringReplicationDataMapper Instance => Lazy.Value; +} + +internal sealed class JsonReplicationDataMapper( + ITypeResolver resolver, + IReplicationDataReader replicationDataReader +): ReplicationDataMapper(replicationDataReader, resolver); diff --git a/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs deleted file mode 100644 index e332b84..0000000 --- a/src/Blumchen/Subscriptions/Replication/ReplicationDataMapper.cs +++ /dev/null @@ -1,198 +0,0 @@ -using System.Text.Json; -using System.Text.Json.Serialization.Metadata; -using Blumchen.Serialization; -using Npgsql; -using Npgsql.Replication.PgOutput; -using Npgsql.Replication.PgOutput.Messages; - -namespace Blumchen.Subscriptions.Replication; - -internal interface IReplicationDataReader -{ - Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default); - Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default); -} - -internal class ObjectReplicationDataReader: IReplicationDataReader -{ - public Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default) - => replicationValue.Get(ct).AsTask(); - - - public Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default) - => reader.GetFieldValueAsync(2, ct); -} - - -internal class StringReplicationDataReader : IReplicationDataReader -{ - public async Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default) - { - using var tr = replicationValue.GetTextReader(); - return await tr.ReadToEndAsync(ct).ConfigureAwait(false); - } - - - public async Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default) - { - using var tr = await reader.GetTextReaderAsync(2, ct).ConfigureAwait(false); - return await tr.ReadToEndAsync(ct).ConfigureAwait(false); - } -} - -internal class JsonReplicationDataReader(JsonTypeResolver resolver): IReplicationDataReader -{ - public async Task Read(ReplicationValue replicationValue, CancellationToken ct, Type? type = default) - { - ArgumentNullException.ThrowIfNull(type); - await using var stream = replicationValue.GetStream(); - return await JsonSerialization.FromJsonAsync(type, stream, resolver.SerializationContext, ct) - .ConfigureAwait(false); - } - - public async Task Read(NpgsqlDataReader reader, CancellationToken ct, Type? type = default) - { - ArgumentNullException.ThrowIfNull(type); - var stream = await reader.GetStreamAsync(2, ct).ConfigureAwait(false); - await using var stream1 = stream.ConfigureAwait(false); - return await JsonSerialization.FromJsonAsync(type, stream, resolver.SerializationContext, ct) - .ConfigureAwait(false); - } -} - -internal class ReplicationDataMapper(IDictionary> mapperSelector) - : IReplicationDataMapper -{ - private readonly Func>, IReplicationJsonBMapper> _memoizer = Memoizer>, IReplicationJsonBMapper>.Execute(SelectMapper); - - private static IReplicationJsonBMapper SelectMapper(string key, - IDictionary> registry) - => registry.TryGetValue(key, out var tuple) - ? tuple.Item1 - : registry.TryGetValue(SubscriptionOptionsBuilder.WildCard, out tuple) - ? tuple.Item1 - : throw new Exception($"Unexpected message `{key}`"); - - public async Task ReadFromReplication(InsertMessage insertMessage, CancellationToken ct) - { - var id = string.Empty; - var columnNumber = 0; - var typeName = string.Empty; - await foreach (var column in insertMessage.NewRow.ConfigureAwait(false)) - { - try - { - switch (columnNumber) - { - case 0: - id = column.Kind == TupleDataKind.BinaryValue - ? (await column.Get(ct).ConfigureAwait(false)).ToString() - : await column.Get(ct).ConfigureAwait(false); - break; - case 1: - using (var textReader = column.GetTextReader()) - { - typeName = await textReader.ReadToEndAsync(ct).ConfigureAwait(false); - break; - } - case 2 when column.GetDataTypeName().Equals("jsonb", StringComparison.OrdinalIgnoreCase): - - return await _memoizer(typeName, mapperSelector).ReadFromReplication(id, typeName, column, ct); - } - } - catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException - or JsonException) - { - return new KoEnvelope(ex, id); - } - - columnNumber++; - } - - throw new InvalidOperationException("You should not get here"); - } - - public async Task ReadFromSnapshot(NpgsqlDataReader reader, CancellationToken ct) - { - long id = default; - try - { - id = reader.GetInt64(0); - var typeName = reader.GetString(1); - - return await mapperSelector[typeName].Item1.ReadFromSnapshot(typeName, id, reader, ct); - } - catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or JsonException) - { - return new KoEnvelope(ex, id.ToString()); - } - } -} - -public interface IReplicationJsonBMapper -{ - Task ReadFromReplication(string id, string typeName, ReplicationValue column, - CancellationToken ct); - - Task ReadFromSnapshot(string typeName, long id, NpgsqlDataReader reader, CancellationToken ct); -} - -internal class ReplicationDataMapper( - IReplicationDataReader replicationDataReader - , ITypeResolver? resolver = default - ): IReplicationJsonBMapper -{ - public async Task ReadFromReplication(string id, string typeName, ReplicationValue column, - CancellationToken ct) - { - - try - { - var type = resolver?.Resolve(typeName); - var value = await replicationDataReader.Read(column, ct, type).ConfigureAwait(false) ?? - throw new ArgumentNullException(); - return new OkEnvelope(value, typeName); - } - catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException - or JsonException) - { - return new KoEnvelope(ex, id); - } - } - - public async Task ReadFromSnapshot(string typeName, long id, NpgsqlDataReader reader, CancellationToken ct) - { - try - { - var eventType = resolver?.Resolve(typeName); - ArgumentNullException.ThrowIfNull(eventType, typeName); - var value = await replicationDataReader.Read(reader, ct, eventType).ConfigureAwait(false) ?? throw new ArgumentNullException(); - return new OkEnvelope(value, typeName); - } - catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or JsonException) - { - return new KoEnvelope(ex, id.ToString()); - } - } -} - -internal sealed class ObjectReplicationDataMapper( - IReplicationDataReader replicationDataReader -): ReplicationDataMapper(replicationDataReader) -{ - private static readonly Lazy Lazy = new(() => new(new ObjectReplicationDataReader())); - public static ObjectReplicationDataMapper Instance => Lazy.Value; -} - -internal sealed class StringReplicationDataMapper( - IReplicationDataReader replicationDataReader -): ReplicationDataMapper(replicationDataReader) -{ - private static readonly Lazy Lazy = new(() => new(new StringReplicationDataReader())); - public static StringReplicationDataMapper Instance => Lazy.Value; -} - -internal sealed class JsonReplicationDataMapper( - ITypeResolver resolver, - IReplicationDataReader replicationDataReader -): ReplicationDataMapper(replicationDataReader, resolver); diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 60ce0e4..2cd5c56 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -1,6 +1,7 @@ using System.Reflection; using System.Runtime.CompilerServices; using Blumchen.Database; +using Blumchen.Subscriber; using Blumchen.Subscriptions.Management; using Blumchen.Subscriptions.Replication; using Npgsql; @@ -14,7 +15,7 @@ namespace Blumchen.Subscriptions; public sealed class Subscription: IAsyncDisposable { private LogicalReplicationConnection? _connection; - private readonly SubscriptionOptionsBuilder _builder = new(); + private readonly OptionsBuilder _builder = new(); private readonly Func _messageHandler = Memoizer Subscribe( - Func builder, + Func builder, [EnumeratorCancellation] CancellationToken ct = default ) { @@ -46,12 +47,12 @@ public async IAsyncEnumerable Subscribe( } internal async IAsyncEnumerable Subscribe( - ISubscriptionOptions subscriptionOptions, + ISubscriberOptions subscriberOptions, [EnumeratorCancellation] CancellationToken ct = default ) { var (dataSource, connectionStringBuilder, publicationSetupOptions, replicationSlotSetupOptions, errorProcessor, - registry) = subscriptionOptions; + registry) = subscriberOptions; await dataSource.EnsureTableExists(publicationSetupOptions.TableDescriptor, ct).ConfigureAwait(false); _connection = new LogicalReplicationConnection(connectionStringBuilder.ConnectionString); @@ -60,7 +61,7 @@ internal async IAsyncEnumerable Subscribe( await dataSource.SetupPublication(publicationSetupOptions, ct).ConfigureAwait(false); var result = await dataSource.SetupReplicationSlot(_connection, replicationSlotSetupOptions, ct) .ConfigureAwait(false); - var replicationDataMapper = new ReplicationDataMapper(registry); + IReplicationDataMapper replicationDataMapper = new ReplicationDataMapper(registry); PgOutputReplicationSlot slot; if (result is not Created created) @@ -132,7 +133,7 @@ IErrorProcessor errorProcessor private static async IAsyncEnumerable ReadExistingRowsFromSnapshot( NpgsqlDataSource dataSource, string snapshotName, - ReplicationDataMapper dataMapper, + IReplicationDataMapper dataMapper, TableDescriptorBuilder.MessageTable tableDescriptor, ISet registeredTypes, [EnumeratorCancellation] CancellationToken ct = default diff --git a/src/Blumchen/MessageTableOptions.cs b/src/Blumchen/TableDescriptorBuilder.cs similarity index 100% rename from src/Blumchen/MessageTableOptions.cs rename to src/Blumchen/TableDescriptorBuilder.cs diff --git a/src/Publisher/Program.cs b/src/Publisher/Program.cs index cdbccc9..5278a96 100644 --- a/src/Publisher/Program.cs +++ b/src/Publisher/Program.cs @@ -1,5 +1,5 @@ using Blumchen.Database; -using Blumchen.Publications; +using Blumchen.Publisher; using Blumchen.Serialization; using Commons; using Npgsql; @@ -19,7 +19,7 @@ var line = Console.ReadLine(); if (line != null && int.TryParse(line, out var result)) { - var resolver = await new PublisherSetupOptionsBuilder() + var resolver = await new OptionsBuilder() .JsonContext(SourceGenerationContext.Default) .NamingPolicy(new AttributeNamingPolicy()) .WithTable(builder => builder.UseDefaults()) //default, but explicit diff --git a/src/SubscriberWorker/MessageHandler.cs b/src/SubscriberWorker/MessageHandler.cs index 7a34383..d9c490e 100644 --- a/src/SubscriberWorker/MessageHandler.cs +++ b/src/SubscriberWorker/MessageHandler.cs @@ -1,4 +1,3 @@ -using Blumchen.Subscriptions; using Blumchen.Subscriptions.Replication; using Microsoft.Extensions.Logging; #pragma warning disable CS9113 // Parameter is unread. diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index 2887539..3ebbdfc 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -4,6 +4,7 @@ using Blumchen; using Blumchen.Database; using Blumchen.Serialization; +using Blumchen.Subscriber; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Management; using Blumchen.Subscriptions.Replication; @@ -67,7 +68,7 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri await command.ExecuteNonQueryAsync(ct); } - protected (TestMessageHandler handler, SubscriptionOptionsBuilder subscriptionOptionsBuilder) SetupFor( + protected (TestMessageHandler handler, OptionsBuilder subscriptionOptionsBuilder) SetupFor( string connectionString, string eventsTable, JsonSerializerContext info, @@ -79,7 +80,7 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri var jsonTypeInfo = info.GetTypeInfo(typeof(T)); ArgumentNullException.ThrowIfNull(jsonTypeInfo); var consumer = new TestMessageHandler(log, jsonTypeInfo); - var subscriptionOptionsBuilder = new SubscriptionOptionsBuilder() + var subscriptionOptionsBuilder = new OptionsBuilder() .WithErrorProcessor(new TestOutErrorProcessor(Output)) .DataSource(new NpgsqlDataSourceBuilder(connectionString).Build()) .ConnectionString(connectionString) diff --git a/src/Tests/When_Subscription_Already_Exists.cs b/src/Tests/When_Subscription_Already_Exists.cs index c6d55e2..7c50f7c 100644 --- a/src/Tests/When_Subscription_Already_Exists.cs +++ b/src/Tests/When_Subscription_Already_Exists.cs @@ -1,4 +1,4 @@ -using Blumchen.Publications; +using Blumchen.Publisher; using Blumchen.Serialization; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Management; @@ -19,7 +19,7 @@ public async Task Read_from_transaction_log() var connectionString = Container.GetConnectionString(); var dataSource = NpgsqlDataSource.Create(connectionString); var eventsTable = await CreateOutboxTable(dataSource, ct); - var opts = new PublisherSetupOptionsBuilder() + var opts = new OptionsBuilder() .JsonContext(PublisherContext.Default) .NamingPolicy(sharedNamingPolicy) .WithTable(o => o.Name(eventsTable)) diff --git a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs index 1528a7b..aad8d93 100644 --- a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs +++ b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs @@ -1,4 +1,4 @@ -using Blumchen.Publications; +using Blumchen.Publisher; using Blumchen.Serialization; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Replication; @@ -18,7 +18,7 @@ public async Task Read_from_table_using_named_transaction_snapshot() var connectionString = Container.GetConnectionString(); var eventsTable = await CreateOutboxTable(NpgsqlDataSource.Create(connectionString), ct); var sharedNamingPolicy = new AttributeNamingPolicy(); - var resolver = new PublisherSetupOptionsBuilder() + var resolver = new OptionsBuilder() .JsonContext(PublisherContext.Default) .NamingPolicy(sharedNamingPolicy) .WithTable(o => o.Name(eventsTable)) diff --git a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs index e8c4f3a..91ca537 100644 --- a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs +++ b/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs @@ -1,4 +1,4 @@ -using Blumchen.Publications; +using Blumchen.Publisher; using Blumchen.Serialization; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Replication; @@ -18,7 +18,7 @@ public async Task Read_from_table_using_named_transaction_snapshot() var connectionString = Container.GetConnectionString(); var eventsTable = await CreateOutboxTable(NpgsqlDataSource.Create(connectionString), ct); - var resolver = new PublisherSetupOptionsBuilder() + var resolver = new OptionsBuilder() .JsonContext(PublisherContext.Default) .NamingPolicy(sharedNamingPolicy) .WithTable(o => o.Name(eventsTable)) From 7729fcd015d79efeff8a67831acfc731f5e8cd16 Mon Sep 17 00:00:00 2001 From: giordanol Date: Sat, 20 Jul 2024 15:05:03 +0200 Subject: [PATCH 42/80] Ensure subscriber default options p --- Blumchen.sln | 6 + src/Blumchen/Blumchen.csproj | 3 + src/Blumchen/Subscriber/OptionsBuilder.cs | 105 ++++++++++++---- .../Management/PublicationManagement.cs | 22 ++-- src/UnitTests/Contracts.cs | 24 ++++ src/UnitTests/UnitTests.csproj | 30 +++++ src/UnitTests/Usings.cs | 2 + src/UnitTests/create_publication.cs | 37 ++++++ src/UnitTests/subscriber_options_builder.cs | 117 ++++++++++++++++++ 9 files changed, 315 insertions(+), 31 deletions(-) create mode 100644 src/UnitTests/Contracts.cs create mode 100644 src/UnitTests/UnitTests.csproj create mode 100644 src/UnitTests/Usings.cs create mode 100644 src/UnitTests/create_publication.cs create mode 100644 src/UnitTests/subscriber_options_builder.cs diff --git a/Blumchen.sln b/Blumchen.sln index f8510ef..50edf3c 100644 --- a/Blumchen.sln +++ b/Blumchen.sln @@ -45,6 +45,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "demo", "demo", "{A4044484-F EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SubscriberWorker", "src\SubscriberWorker\SubscriberWorker.csproj", "{DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTests", "src\UnitTests\UnitTests.csproj", "{B16305B4-8AC3-4435-AADB-D9E2ACAA1C13}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -75,6 +77,10 @@ Global {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}.Debug|Any CPU.Build.0 = Debug|Any CPU {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}.Release|Any CPU.ActiveCfg = Release|Any CPU {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}.Release|Any CPU.Build.0 = Release|Any CPU + {B16305B4-8AC3-4435-AADB-D9E2ACAA1C13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B16305B4-8AC3-4435-AADB-D9E2ACAA1C13}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B16305B4-8AC3-4435-AADB-D9E2ACAA1C13}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B16305B4-8AC3-4435-AADB-D9E2ACAA1C13}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Blumchen/Blumchen.csproj b/src/Blumchen/Blumchen.csproj index 0138080..d275b6b 100644 --- a/src/Blumchen/Blumchen.csproj +++ b/src/Blumchen/Blumchen.csproj @@ -39,6 +39,9 @@ <_Parameter1>Tests + + <_Parameter1>UnitTests + <_Parameter1>Blumchen.DependencyInjection diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index ae86d0b..1056afd 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -1,3 +1,5 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; using System.Text.Json.Serialization; using Blumchen.Serialization; using Blumchen.Subscriptions; @@ -11,18 +13,28 @@ namespace Blumchen.Subscriber; public sealed class OptionsBuilder { internal const string WildCard = "*"; - private NpgsqlConnectionStringBuilder? _connectionStringBuilder; - private NpgsqlDataSource? _dataSource; private PublicationManagement.PublicationSetupOptions _publicationSetupOptions = new(); private ReplicationSlotManagement.ReplicationSlotSetupOptions? _replicationSlotSetupOptions; + + [System.Diagnostics.CodeAnalysis.NotNull] + private NpgsqlConnectionStringBuilder? _connectionStringBuilder = default; + + [System.Diagnostics.CodeAnalysis.NotNull] + private NpgsqlDataSource? _dataSource = default; + private readonly Dictionary _typeRegistry = []; - private readonly Dictionary> _replicationDataMapperSelector = []; + + private readonly Dictionary> + _replicationDataMapperSelector = []; + private IErrorProcessor? _errorProcessor; private INamingPolicy? _namingPolicy; private readonly TableDescriptorBuilder _tableDescriptorBuilder = new(); private TableDescriptorBuilder.MessageTable? _messageTable; - - private readonly IReplicationJsonBMapper _objectDataMapper = new ObjectReplicationDataMapper(new ObjectReplicationDataReader()); + + private readonly IReplicationJsonBMapper _objectDataMapper = + new ObjectReplicationDataMapper(new ObjectReplicationDataReader()); + private IReplicationJsonBMapper? _jsonDataMapper; private JsonSerializerContext? _jsonSerializerContext; @@ -38,6 +50,7 @@ public OptionsBuilder WithTable( [UsedImplicitly] public OptionsBuilder ConnectionString(string connectionString) { + Ensure.Null(_connectionStringBuilder, nameof(ConnectionString)); _connectionStringBuilder = new NpgsqlConnectionStringBuilder(connectionString); return this; } @@ -45,6 +58,7 @@ public OptionsBuilder ConnectionString(string connectionString) [UsedImplicitly] public OptionsBuilder DataSource(NpgsqlDataSource dataSource) { + Ensure.Null(_dataSource, nameof(DataSource)); _dataSource = dataSource; return this; } @@ -67,12 +81,13 @@ public OptionsBuilder JsonContext(JsonSerializerContext jsonSerializerContext) public OptionsBuilder WithPublicationOptions(PublicationManagement.PublicationSetupOptions publicationOptions) { _publicationSetupOptions = - publicationOptions with { RegisteredTypes = _publicationSetupOptions.RegisteredTypes}; + publicationOptions with { RegisteredTypes = _publicationSetupOptions.RegisteredTypes }; return this; } [UsedImplicitly] - public OptionsBuilder WithReplicationOptions(ReplicationSlotManagement.ReplicationSlotSetupOptions replicationSlotOptions) + public OptionsBuilder WithReplicationOptions( + ReplicationSlotManagement.ReplicationSlotSetupOptions replicationSlotOptions) { _replicationSlotSetupOptions = replicationSlotOptions; return this; @@ -96,25 +111,32 @@ public OptionsBuilder ConsumesRowString(IMessageHandler handler) wher [UsedImplicitly] public OptionsBuilder ConsumesRowStrings(IMessageHandler handler) { - _replicationDataMapperSelector.Add(WildCard, new Tuple(StringReplicationDataMapper.Instance, handler)); + _replicationDataMapperSelector.Add(WildCard, + new Tuple(StringReplicationDataMapper.Instance, handler)); return this; } [UsedImplicitly] public OptionsBuilder ConsumesRowObjects(IMessageHandler handler) { - _replicationDataMapperSelector.Add(WildCard, new Tuple(ObjectReplicationDataMapper.Instance, handler)); + _replicationDataMapperSelector.Add(WildCard, + new Tuple(ObjectReplicationDataMapper.Instance, handler)); return this; } - private OptionsBuilder ConsumesRow(IMessageHandler handler, RawUrnAttribute.RawData filter, IReplicationJsonBMapper dataMapper) where T : class + private OptionsBuilder ConsumesRow(IMessageHandler handler, RawUrnAttribute.RawData filter, + IReplicationJsonBMapper dataMapper) where T : class { - using var urnEnum = typeof(T) + var urns = typeof(T) .GetCustomAttributes(typeof(RawUrnAttribute), false) .OfType() .Where(attribute => attribute.Data == filter) - .Select(attribute => attribute.Urn).GetEnumerator(); - while (urnEnum.MoveNext()) _replicationDataMapperSelector.Add(urnEnum.Current.ToString(), new Tuple(dataMapper, handler)); + .Select(attribute => attribute.Urn).ToList(); + Ensure.NotEmpty>(urns, nameof(NamingPolicy)); + using var urnEnum = urns.GetEnumerator(); + while (urnEnum.MoveNext()) + _replicationDataMapperSelector.Add(urnEnum.Current.ToString(), + new Tuple(dataMapper, handler)); return this; } @@ -124,35 +146,70 @@ public OptionsBuilder WithErrorProcessor(IErrorProcessor? errorProcessor) _errorProcessor = errorProcessor; return this; } - + + internal abstract class Validable(Func condition, string errorFormat) + { + public void IsValid(T value, params object[] parameters) + { + if (!condition(value)) throw new ConfigurationException(string.Format(errorFormat, parameters)); + } + } + + internal static class Ensure + { + public static void Null(T value, params object[] parameters) => + new NullTrait().IsValid(value, parameters); + + public static void NotNull(T value, params object[] parameters) => + new NotNullTrait().IsValid(value, parameters); + + public static void NotEmpty(T value, params object[] parameters) => + new NotEmptyTrait().IsValid(value, parameters); + } + + internal class NullTrait() + : Validable(v => v is null, $"`{{0}}` method on {nameof(OptionsBuilder)} called more then once"); + + internal class NotNullTrait() + : Validable(v => v is not null, $"`{{0}}` method not called on {nameof(OptionsBuilder)}"); + + internal class NotEmptyTrait(): Validable(v => v is ICollection { Count: > 0 }, + $"No `{{0}}` method called on {nameof(OptionsBuilder)}"); + internal ISubscriberOptions Build() { _messageTable ??= _tableDescriptorBuilder.Build(); - ArgumentNullException.ThrowIfNull(_connectionStringBuilder); - ArgumentNullException.ThrowIfNull(_dataSource); - - if(_typeRegistry.Count > 0) + Ensure.NotNull(_connectionStringBuilder, $"{nameof(ConnectionString)}"); + Ensure.NotNull(_dataSource, $"{nameof(DataSource)}"); + + if (_typeRegistry.Count > 0) { + Ensure.NotNull(_namingPolicy, $"{nameof(NamingPolicy)}"); if (_jsonSerializerContext != null) { var typeResolver = new JsonTypeResolver(_jsonSerializerContext, _namingPolicy); foreach (var type in _typeRegistry.Keys) typeResolver.WhiteList(type); - _jsonDataMapper = new JsonReplicationDataMapper(typeResolver, new JsonReplicationDataReader(typeResolver)); - - foreach (var (key, value) in typeResolver.RegisteredTypes.Join(_typeRegistry,pair => pair.Value, pair => pair.Key, (pair, valuePair) => (pair.Key, valuePair.Value))) - _replicationDataMapperSelector.Add(key,new Tuple(_jsonDataMapper, value)); + _jsonDataMapper = + new JsonReplicationDataMapper(typeResolver, new JsonReplicationDataReader(typeResolver)); + + foreach (var (key, value) in typeResolver.RegisteredTypes.Join(_typeRegistry, pair => pair.Value, + pair => pair.Key, (pair, valuePair) => (pair.Key, valuePair.Value))) + _replicationDataMapperSelector.Add(key, + new Tuple(_jsonDataMapper, value)); } else { throw new ConfigurationException($"`${nameof(Consumes)}<>` requires a valid `{nameof(JsonContext)}`."); } } + + Ensure.NotEmpty(_replicationDataMapperSelector, $"{nameof(Consumes)}..."); _publicationSetupOptions = _publicationSetupOptions with { - RegisteredTypes = _replicationDataMapperSelector.Keys.Except(new [] { WildCard }).ToHashSet(), + RegisteredTypes = _replicationDataMapperSelector.Keys.Except([WildCard]).ToHashSet(), TableDescriptor = _messageTable }; return new SubscriberOptions( @@ -162,7 +219,7 @@ internal ISubscriberOptions Build() _replicationSlotSetupOptions ?? new ReplicationSlotManagement.ReplicationSlotSetupOptions(), _errorProcessor ?? new ConsoleOutErrorProcessor(), _replicationDataMapperSelector - ); + ); } } diff --git a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs index 0a9ff27..5e044ad 100644 --- a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs +++ b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs @@ -59,22 +59,30 @@ CancellationToken ct } } - internal static Task CreatePublication( - this NpgsqlDataSource dataSource, + internal static string CreatePublication( string publicationName, string tableName, - ISet registeredTypes, - CancellationToken ct - ) { + ISet registeredTypes + ) + { var sql = $"CREATE PUBLICATION \"{publicationName}\" FOR TABLE {tableName} {{0}} WITH (publish = 'insert');"; return registeredTypes.Count switch { - 0 => Execute(dataSource, string.Format(sql,string.Empty), ct), - _ => Execute(dataSource, string.Format(sql, $"WHERE ({PublicationFilter(registeredTypes)})"), ct) + 0 => string.Format(sql, string.Empty), + _ => string.Format(sql, $"WHERE ({PublicationFilter(registeredTypes)})") }; static string PublicationFilter(ICollection input) => string.Join(" OR ", input.Select(s => $"message_type = '{s}'")); } + internal static Task CreatePublication( + this NpgsqlDataSource dataSource, + string publicationName, + string tableName, + ISet registeredTypes, + CancellationToken ct + ) => Execute(dataSource, CreatePublication(publicationName, tableName, registeredTypes), ct); + + private static async Task Execute( this NpgsqlDataSource dataSource, string sql, diff --git a/src/UnitTests/Contracts.cs b/src/UnitTests/Contracts.cs new file mode 100644 index 0000000..8e7191a --- /dev/null +++ b/src/UnitTests/Contracts.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; +using Blumchen.Serialization; + +namespace UnitTests +{ + [MessageUrn("user-created:v1")] + public record UserCreatedContract( + Guid Id, + string Name + ); + + [RawUrn("user-deleted:v1", RawUrnAttribute.RawData.Object)] + public class MessageObjects; + + + [RawUrn("user-modified:v1", RawUrnAttribute.RawData.String)] + internal class MessageString; + + internal class InvalidMessage; + + [JsonSourceGenerationOptions(WriteIndented = true)] + [JsonSerializable(typeof(UserCreatedContract))] + internal partial class SourceGenerationContext: JsonSerializerContext; +} diff --git a/src/UnitTests/UnitTests.csproj b/src/UnitTests/UnitTests.csproj new file mode 100644 index 0000000..96ce6e4 --- /dev/null +++ b/src/UnitTests/UnitTests.csproj @@ -0,0 +1,30 @@ + + + + net8.0 + enable + enable + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + diff --git a/src/UnitTests/Usings.cs b/src/UnitTests/Usings.cs new file mode 100644 index 0000000..2af8a54 --- /dev/null +++ b/src/UnitTests/Usings.cs @@ -0,0 +1,2 @@ +global using Xunit; + diff --git a/src/UnitTests/create_publication.cs b/src/UnitTests/create_publication.cs new file mode 100644 index 0000000..59616f8 --- /dev/null +++ b/src/UnitTests/create_publication.cs @@ -0,0 +1,37 @@ +using static Blumchen.Subscriptions.Management.PublicationManagement; + +namespace UnitTests +{ + public class create_publication + { + [Fact] + public void with_no_publication_filter() + { + var publicationName = "publicationName"; + var tableName = "tableName"; + var sql = $"CREATE PUBLICATION \"{publicationName}\" FOR TABLE {tableName} WITH (publish = 'insert');"; + Assert.Equal(sql, CreatePublication(publicationName, tableName, new HashSet())); + } + + [Fact] + public void with_single_publication_filter() + { + const string publicationName = "publicationName"; + const string tableName = "tableName"; + const string messageType = "messageType"; + var sql = $"CREATE PUBLICATION \"{publicationName}\" FOR TABLE {tableName} WHERE (message_type = '{messageType}') WITH (publish = 'insert');"; + Assert.Equal(sql, CreatePublication(publicationName, tableName, new HashSet { messageType })); + } + + [Fact] + public void with_multiple_publication_filters() + { + const string publicationName = "publicationName"; + const string tableName = "tableName"; + const string messageType1 = "messageType1"; + const string messageType2 = "messageType2"; + var sql = $"CREATE PUBLICATION \"{publicationName}\" FOR TABLE {tableName} WHERE (message_type = '{messageType1}' OR message_type = '{messageType2}') WITH (publish = 'insert');"; + Assert.Equal(sql, CreatePublication(publicationName, tableName, new HashSet { messageType1, messageType2 })); + } + } +} diff --git a/src/UnitTests/subscriber_options_builder.cs b/src/UnitTests/subscriber_options_builder.cs new file mode 100644 index 0000000..61df25d --- /dev/null +++ b/src/UnitTests/subscriber_options_builder.cs @@ -0,0 +1,117 @@ +using Blumchen; +using Blumchen.Subscriber; +using Blumchen.Subscriptions; +using Blumchen.Subscriptions.Replication; +using Npgsql; +using NSubstitute; +using static Blumchen.Subscriptions.Management.PublicationManagement; +using static Blumchen.Subscriptions.Subscription; + +namespace UnitTests +{ + public class subscriber_options_builder + { + private const string ValidConnectionString = + "PORT = 5432; HOST = 127.0.0.1; TIMEOUT = 15; MINPOOLSIZE = 1; MAXPOOLSIZE = 100; COMMANDTIMEOUT = 20; Include Error Detail=True; DATABASE = 'postgres'; PASSWORD = 'postgres'; USER ID = 'postgres';"; + private readonly Func _builder = c => new OptionsBuilder().ConnectionString(c).DataSource(new NpgsqlDataSourceBuilder(c).Build()); + + [Fact] + public void requires_at_least_one_method_call_to_connectionstring() + { + var exception = Record.Exception(() => new OptionsBuilder().Build()); + Assert.IsType(exception); + Assert.Equal("`ConnectionString` method not called on OptionsBuilder", exception.Message); + } + + [Fact] + public void requires_at_most_one_method_call_to_connectionstring() + { + var exception = Record.Exception(() => new OptionsBuilder() + .ConnectionString(ValidConnectionString) + .ConnectionString(ValidConnectionString) + .Build()); + Assert.IsType(exception); + Assert.Equal("`ConnectionString` method on OptionsBuilder called more then once", exception.Message); + } + + [Fact] + public void requires_at_least_one_method_call_to_datasource() + { + var exception = Record.Exception(() => new OptionsBuilder().ConnectionString(ValidConnectionString).Build()); + Assert.IsType(exception); + Assert.Equal("`DataSource` method not called on OptionsBuilder", exception.Message); + } + + [Fact] + public void requires_at_most_one_method_call_to_datasource() + { + var exception = Record.Exception(() => _builder(ValidConnectionString).DataSource(new NpgsqlDataSourceBuilder(ValidConnectionString).Build()).Build()); + Assert.IsType(exception); + Assert.Equal("`DataSource` method on OptionsBuilder called more then once", exception.Message); + } + + + [Fact] + public void requires_at_least_one_method_call_to_consumes() + { + var exception = Record.Exception(() => _builder(ValidConnectionString).Build()); + Assert.IsType(exception); + Assert.Equal("No `Consumes...` method called on OptionsBuilder", exception.Message); + } + + [Fact] + public void has_default_options() + { + var messageHandler = Substitute.For>(); + var opts = _builder(ValidConnectionString).ConsumesRowStrings(messageHandler).Build(); + + Assert.NotNull(opts.PublicationOptions); + Assert.Equal(CreateStyle.WhenNotExists, opts.PublicationOptions.CreateStyle); + Assert.False(opts.PublicationOptions.ShouldReAddTablesIfWereRecreated); + Assert.Empty(opts.PublicationOptions.RegisteredTypes); + Assert.Equal(opts.PublicationOptions.PublicationName, opts.PublicationOptions.PublicationName); + Assert.Equal(new TableDescriptorBuilder().Build(), opts.PublicationOptions.TableDescriptor); + + Assert.NotNull(opts.ReplicationOptions); + Assert.Equal($"{TableDescriptorBuilder.MessageTable.DefaultName}_slot", opts.ReplicationOptions.SlotName); + Assert.Equal(CreateStyle.WhenNotExists, opts.ReplicationOptions.CreateStyle); + Assert.False(opts.ReplicationOptions.Binary); + + Assert.IsType(opts.ErrorProcessor); + } + + [Fact] + public void with_ConsumesRowStrings() + { + var messageHandler = Substitute.For>(); + var opts = _builder(ValidConnectionString).ConsumesRowStrings(messageHandler).Build(); + Assert.Equivalent(new Dictionary> { { OptionsBuilder.WildCard, new Tuple(StringReplicationDataMapper.Instance, messageHandler) } }, opts.Registry); + } + + [Fact] + public void with_ConsumesRowObjects() + { + var messageHandler = Substitute.For>(); + var opts = _builder(ValidConnectionString).ConsumesRowObjects(messageHandler).Build(); + Assert.Equivalent(new Dictionary> { { OptionsBuilder.WildCard, new Tuple(ObjectReplicationDataMapper.Instance, messageHandler) } }, opts.Registry); + } + + [Fact] + public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() + { + var messageHandler = Substitute.For>(); + var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRowObject(messageHandler).Build()); + Assert.IsType(exception); + Assert.Equal("No `NamingPolicy` method called on OptionsBuilder", exception.Message); + } + + [Fact] + public void with_typed_raw_consumer_of_string_requires_RawUrn_decoration() + { + var messageHandler = Substitute.For>(); + var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRowString(messageHandler).Build()); + Assert.IsType(exception); + Assert.Equal("No `NamingPolicy` method called on OptionsBuilder", exception.Message); + } + } +} From 5ae673eb9e47c4b4030288aa9eb37f435f85037d Mon Sep 17 00:00:00 2001 From: giordanol Date: Sat, 20 Jul 2024 15:08:29 +0200 Subject: [PATCH 43/80] rename `PublicationSetupOptions` to `PublicationOptions` and `ReplicationSlotSetupOptions` to `ReplicationSlotOptions` --- src/Blumchen/Subscriber/ISubscriberOptions.cs | 12 ++++---- src/Blumchen/Subscriber/OptionsBuilder.cs | 30 +++++++------------ .../Management/PublicationManagement.cs | 12 ++++---- .../Management/ReplicationSlotManagement.cs | 4 +-- src/SubscriberWorker/Program.cs | 8 ++--- src/Tests/DatabaseFixture.cs | 4 +-- 6 files changed, 31 insertions(+), 39 deletions(-) diff --git a/src/Blumchen/Subscriber/ISubscriberOptions.cs b/src/Blumchen/Subscriber/ISubscriberOptions.cs index caccd5a..3c0e3f6 100644 --- a/src/Blumchen/Subscriber/ISubscriberOptions.cs +++ b/src/Blumchen/Subscriber/ISubscriberOptions.cs @@ -12,15 +12,15 @@ public interface ISubscriberOptions [UsedImplicitly] NpgsqlDataSource DataSource { get; } [UsedImplicitly] NpgsqlConnectionStringBuilder ConnectionStringBuilder { get; } IDictionary> Registry { get; } - [UsedImplicitly] PublicationSetupOptions PublicationOptions { get; } - [UsedImplicitly] ReplicationSlotSetupOptions ReplicationOptions { get; } + [UsedImplicitly] PublicationOptions PublicationOptions { get; } + [UsedImplicitly] ReplicationSlotOptions ReplicationOptions { get; } [UsedImplicitly] IErrorProcessor ErrorProcessor { get; } void Deconstruct( out NpgsqlDataSource dataSource, out NpgsqlConnectionStringBuilder connectionStringBuilder, - out PublicationSetupOptions publicationSetupOptions, - out ReplicationSlotSetupOptions replicationSlotSetupOptions, + out PublicationOptions publicationOptions, + out ReplicationSlotOptions replicationSlotOptions, out IErrorProcessor errorProcessor, out IDictionary> registry); } @@ -28,7 +28,7 @@ void Deconstruct( internal record SubscriberOptions( NpgsqlDataSource DataSource, NpgsqlConnectionStringBuilder ConnectionStringBuilder, - PublicationSetupOptions PublicationOptions, - ReplicationSlotSetupOptions ReplicationOptions, + PublicationOptions PublicationOptions, + ReplicationSlotOptions ReplicationOptions, IErrorProcessor ErrorProcessor, IDictionary> Registry): ISubscriberOptions; diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index 1056afd..2b6f4d2 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -1,5 +1,4 @@ using System.Collections; -using System.Diagnostics.CodeAnalysis; using System.Text.Json.Serialization; using Blumchen.Serialization; using Blumchen.Subscriptions; @@ -13,20 +12,15 @@ namespace Blumchen.Subscriber; public sealed class OptionsBuilder { internal const string WildCard = "*"; - private PublicationManagement.PublicationSetupOptions _publicationSetupOptions = new(); - private ReplicationSlotManagement.ReplicationSlotSetupOptions? _replicationSlotSetupOptions; - [System.Diagnostics.CodeAnalysis.NotNull] private NpgsqlConnectionStringBuilder? _connectionStringBuilder = default; - [System.Diagnostics.CodeAnalysis.NotNull] private NpgsqlDataSource? _dataSource = default; - + private PublicationManagement.PublicationOptions _publicationOptions = new(); + private ReplicationSlotManagement.ReplicationSlotOptions? _replicationSlotOptions; private readonly Dictionary _typeRegistry = []; - private readonly Dictionary> _replicationDataMapperSelector = []; - private IErrorProcessor? _errorProcessor; private INamingPolicy? _namingPolicy; private readonly TableDescriptorBuilder _tableDescriptorBuilder = new(); @@ -78,18 +72,18 @@ public OptionsBuilder JsonContext(JsonSerializerContext jsonSerializerContext) } [UsedImplicitly] - public OptionsBuilder WithPublicationOptions(PublicationManagement.PublicationSetupOptions publicationOptions) + public OptionsBuilder WithPublicationOptions(PublicationManagement.PublicationOptions publicationOptions) { - _publicationSetupOptions = - publicationOptions with { RegisteredTypes = _publicationSetupOptions.RegisteredTypes }; + + _publicationOptions = + publicationOptions with { RegisteredTypes = _publicationOptions.RegisteredTypes}; return this; } [UsedImplicitly] - public OptionsBuilder WithReplicationOptions( - ReplicationSlotManagement.ReplicationSlotSetupOptions replicationSlotOptions) + public OptionsBuilder WithReplicationOptions(ReplicationSlotManagement.ReplicationSlotOptions replicationSlotOptions) { - _replicationSlotSetupOptions = replicationSlotOptions; + _replicationSlotOptions = replicationSlotOptions; return this; } @@ -204,9 +198,7 @@ internal ISubscriberOptions Build() throw new ConfigurationException($"`${nameof(Consumes)}<>` requires a valid `{nameof(JsonContext)}`."); } } - - Ensure.NotEmpty(_replicationDataMapperSelector, $"{nameof(Consumes)}..."); - _publicationSetupOptions = _publicationSetupOptions + _publicationOptions = _publicationOptions with { RegisteredTypes = _replicationDataMapperSelector.Keys.Except([WildCard]).ToHashSet(), @@ -215,8 +207,8 @@ internal ISubscriberOptions Build() return new SubscriberOptions( _dataSource, _connectionStringBuilder, - _publicationSetupOptions, - _replicationSlotSetupOptions ?? new ReplicationSlotManagement.ReplicationSlotSetupOptions(), + _publicationOptions, + _replicationSlotOptions ?? new ReplicationSlotManagement.ReplicationSlotOptions(), _errorProcessor ?? new ConsoleOutErrorProcessor(), _replicationDataMapperSelector ); diff --git a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs index 5e044ad..2f667f3 100644 --- a/src/Blumchen/Subscriptions/Management/PublicationManagement.cs +++ b/src/Blumchen/Subscriptions/Management/PublicationManagement.cs @@ -11,11 +11,11 @@ public static class PublicationManagement { public static async Task SetupPublication( this NpgsqlDataSource dataSource, - PublicationSetupOptions setupOptions, + PublicationOptions options, CancellationToken ct ) { - var (publicationName, createStyle, shouldReAddTablesIfWereRecreated, registeredTypes, tableDescription) = setupOptions; + var (publicationName, createStyle, shouldReAddTablesIfWereRecreated, registeredTypes, tableDescription) = options; return createStyle switch { @@ -23,7 +23,7 @@ CancellationToken ct Subscription.CreateStyle.AlwaysRecreate => await ReCreate(dataSource, publicationName, tableDescription.Name, registeredTypes, ct).ConfigureAwait(false), Subscription.CreateStyle.WhenNotExists when await dataSource.PublicationExists(publicationName, ct).ConfigureAwait(false) => await Refresh(dataSource, publicationName, tableDescription.Name, shouldReAddTablesIfWereRecreated, ct).ConfigureAwait(false), Subscription.CreateStyle.WhenNotExists => await Create(dataSource, publicationName, tableDescription.Name, registeredTypes, ct).ConfigureAwait(false), - _ => throw new ArgumentOutOfRangeException(nameof(setupOptions.CreateStyle)) + _ => throw new ArgumentOutOfRangeException(nameof(options.CreateStyle)) }; static async Task ReCreate( @@ -141,13 +141,13 @@ public record AlreadyExists: SetupPublicationResult; public record Created: SetupPublicationResult; } - public sealed record PublicationSetupOptions( - string PublicationName = PublicationSetupOptions.DefaultPublicationName, + public sealed record PublicationOptions( + string PublicationName = PublicationOptions.DefaultName, Subscription.CreateStyle CreateStyle = Subscription.CreateStyle.WhenNotExists, bool ShouldReAddTablesIfWereRecreated = false ) { - internal const string DefaultPublicationName = "pub"; + internal const string DefaultName = "pub"; internal ISet RegisteredTypes { get; init; } = Enumerable.Empty().ToHashSet(); internal TableDescriptorBuilder.MessageTable TableDescriptor { get; init; } = new TableDescriptorBuilder().Build(); diff --git a/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs b/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs index adeb6a8..f95fb3a 100644 --- a/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs +++ b/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs @@ -18,7 +18,7 @@ CancellationToken ct public static async Task SetupReplicationSlot( this NpgsqlDataSource dataSource, LogicalReplicationConnection connection, - ReplicationSlotSetupOptions options, + ReplicationSlotOptions options, CancellationToken ct ) { @@ -59,7 +59,7 @@ static async Task Create( } } - public record ReplicationSlotSetupOptions( + public record ReplicationSlotOptions( string SlotName = $"{TableDescriptorBuilder.MessageTable.DefaultName}_slot", Subscription.CreateStyle CreateStyle = Subscription.CreateStyle.WhenNotExists, bool Binary = diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index 5a3d68e..6d3b91b 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -63,8 +63,8 @@ subscriptionOptions .ConnectionString(Settings.ConnectionString) .DataSource(provider.GetRequiredService()) - .WithReplicationOptions(new ReplicationSlotManagement.ReplicationSlotSetupOptions($"{nameof(HandleImpl1)}_slot")) - .WithPublicationOptions(new PublicationManagement.PublicationSetupOptions($"{nameof(HandleImpl1)}_pub")) + .WithReplicationOptions(new ReplicationSlotManagement.ReplicationSlotOptions($"{nameof(HandleImpl1)}_slot")) + .WithPublicationOptions(new PublicationManagement.PublicationOptions($"{nameof(HandleImpl1)}_pub")) .WithErrorProcessor(provider.GetRequiredService()) .NamingPolicy(provider.GetRequiredService()) .JsonContext(SourceGenerationContext.Default) @@ -78,8 +78,8 @@ .Subscription(subscriptionOptions => subscriptionOptions.ConnectionString(Settings.ConnectionString) .DataSource(provider.GetRequiredService()) - .WithReplicationOptions(new ReplicationSlotManagement.ReplicationSlotSetupOptions($"{nameof(HandleImpl2)}_slot")) - .WithPublicationOptions(new PublicationManagement.PublicationSetupOptions($"{nameof(HandleImpl2)}_pub")) + .WithReplicationOptions(new ReplicationSlotManagement.ReplicationSlotOptions($"{nameof(HandleImpl2)}_slot")) + .WithPublicationOptions(new PublicationManagement.PublicationOptions($"{nameof(HandleImpl2)}_pub")) .WithErrorProcessor(provider.GetRequiredService()) .NamingPolicy(provider.GetRequiredService()) .JsonContext(SourceGenerationContext.Default) diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index 3ebbdfc..06664c0 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -89,10 +89,10 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri .Consumes(consumer) .WithTable(o => o.Name(eventsTable)) .WithPublicationOptions( - new PublicationManagement.PublicationSetupOptions(PublicationName: publicationName ?? Randomise("events_pub")) + new PublicationManagement.PublicationOptions(PublicationName: publicationName ?? Randomise("events_pub")) ) .WithReplicationOptions( - new ReplicationSlotManagement.ReplicationSlotSetupOptions(slotName ?? Randomise("events_slot")) + new ReplicationSlotManagement.ReplicationSlotOptions(slotName ?? Randomise("events_slot")) ); return (consumer, subscriptionOptionsBuilder); } From 85f520b821ec904974dc542d878874ff3c509d69 Mon Sep 17 00:00:00 2001 From: giordanol Date: Sat, 20 Jul 2024 16:29:38 +0200 Subject: [PATCH 44/80] move classes to files --- src/Blumchen/ConfigurationException.cs | 3 ++ src/Blumchen/Subscriber/Ensure.cs | 23 +++++++++++++ src/Blumchen/Subscriber/OptionsBuilder.cs | 42 +++++------------------ 3 files changed, 35 insertions(+), 33 deletions(-) create mode 100644 src/Blumchen/ConfigurationException.cs create mode 100644 src/Blumchen/Subscriber/Ensure.cs diff --git a/src/Blumchen/ConfigurationException.cs b/src/Blumchen/ConfigurationException.cs new file mode 100644 index 0000000..93a24f7 --- /dev/null +++ b/src/Blumchen/ConfigurationException.cs @@ -0,0 +1,3 @@ +namespace Blumchen; + +public class ConfigurationException(string message): Exception(message); diff --git a/src/Blumchen/Subscriber/Ensure.cs b/src/Blumchen/Subscriber/Ensure.cs new file mode 100644 index 0000000..22edc28 --- /dev/null +++ b/src/Blumchen/Subscriber/Ensure.cs @@ -0,0 +1,23 @@ +using System.Collections; + +namespace Blumchen.Subscriber; + +internal static class Ensure +{ + public static void Null(T value, params object[] parameters) => new NullTrait().IsValid(value, parameters); + public static void NotNull(T value, params object[] parameters) => new NotNullTrait().IsValid(value, parameters); + public static void NotEmpty(T value, params object[] parameters) => new NotEmptyTrait().IsValid(value, parameters); +} + +internal abstract class Validable(Func condition, string errorFormat) +{ + public void IsValid(T value, params object[] parameters) + { + if (!condition(value)) + throw new ConfigurationException(string.Format(errorFormat, parameters)); + } +} + +internal class NullTrait(): Validable(v => v is null, $"`{{0}}` method on {nameof(OptionsBuilder)} called more then once"); +internal class NotNullTrait(): Validable(v => v is not null, $"`{{0}}` method not called on {nameof(OptionsBuilder)}"); +internal class NotEmptyTrait(): Validable(v => v is ICollection { Count: > 0 }, $"No `{{0}}` method called on {nameof(OptionsBuilder)}"); diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index 2b6f4d2..b09cc61 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -12,15 +12,20 @@ namespace Blumchen.Subscriber; public sealed class OptionsBuilder { internal const string WildCard = "*"; + [System.Diagnostics.CodeAnalysis.NotNull] private NpgsqlConnectionStringBuilder? _connectionStringBuilder = default; + [System.Diagnostics.CodeAnalysis.NotNull] private NpgsqlDataSource? _dataSource = default; + private PublicationManagement.PublicationOptions _publicationOptions = new(); private ReplicationSlotManagement.ReplicationSlotOptions? _replicationSlotOptions; private readonly Dictionary _typeRegistry = []; + private readonly Dictionary> _replicationDataMapperSelector = []; + private IErrorProcessor? _errorProcessor; private INamingPolicy? _namingPolicy; private readonly TableDescriptorBuilder _tableDescriptorBuilder = new(); @@ -76,12 +81,13 @@ public OptionsBuilder WithPublicationOptions(PublicationManagement.PublicationOp { _publicationOptions = - publicationOptions with { RegisteredTypes = _publicationOptions.RegisteredTypes}; + publicationOptions with { RegisteredTypes = _publicationOptions.RegisteredTypes }; return this; } [UsedImplicitly] - public OptionsBuilder WithReplicationOptions(ReplicationSlotManagement.ReplicationSlotOptions replicationSlotOptions) + public OptionsBuilder WithReplicationOptions( + ReplicationSlotManagement.ReplicationSlotOptions replicationSlotOptions) { _replicationSlotOptions = replicationSlotOptions; return this; @@ -141,35 +147,6 @@ public OptionsBuilder WithErrorProcessor(IErrorProcessor? errorProcessor) return this; } - internal abstract class Validable(Func condition, string errorFormat) - { - public void IsValid(T value, params object[] parameters) - { - if (!condition(value)) throw new ConfigurationException(string.Format(errorFormat, parameters)); - } - } - - internal static class Ensure - { - public static void Null(T value, params object[] parameters) => - new NullTrait().IsValid(value, parameters); - - public static void NotNull(T value, params object[] parameters) => - new NotNullTrait().IsValid(value, parameters); - - public static void NotEmpty(T value, params object[] parameters) => - new NotEmptyTrait().IsValid(value, parameters); - } - - internal class NullTrait() - : Validable(v => v is null, $"`{{0}}` method on {nameof(OptionsBuilder)} called more then once"); - - internal class NotNullTrait() - : Validable(v => v is not null, $"`{{0}}` method not called on {nameof(OptionsBuilder)}"); - - internal class NotEmptyTrait(): Validable(v => v is ICollection { Count: > 0 }, - $"No `{{0}}` method called on {nameof(OptionsBuilder)}"); - internal ISubscriberOptions Build() { _messageTable ??= _tableDescriptorBuilder.Build(); @@ -198,6 +175,7 @@ internal ISubscriberOptions Build() throw new ConfigurationException($"`${nameof(Consumes)}<>` requires a valid `{nameof(JsonContext)}`."); } } + _publicationOptions = _publicationOptions with { @@ -214,5 +192,3 @@ internal ISubscriberOptions Build() ); } } - -public class ConfigurationException(string message): Exception(message); From 20bc581b3ccf0baf324e4318dc0da4d0a7e23377 Mon Sep 17 00:00:00 2001 From: giordanol Date: Sun, 21 Jul 2024 09:34:24 +0200 Subject: [PATCH 45/80] renamed files --- ...on_Already_Exists.cs => if_subscription_already_exists.cs} | 4 ++-- ...s => if_subscription_does_not_exist_and_table_is_empty.cs} | 4 ++-- ... if_subscription_does_not_exist_and_table_is_not_empty.cs} | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) rename src/Tests/{When_Subscription_Already_Exists.cs => if_subscription_already_exists.cs} (92%) rename src/Tests/{When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs => if_subscription_does_not_exist_and_table_is_empty.cs} (90%) rename src/Tests/{When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs => if_subscription_does_not_exist_and_table_is_not_empty.cs} (89%) diff --git a/src/Tests/When_Subscription_Already_Exists.cs b/src/Tests/if_subscription_already_exists.cs similarity index 92% rename from src/Tests/When_Subscription_Already_Exists.cs rename to src/Tests/if_subscription_already_exists.cs index 7c50f7c..875a487 100644 --- a/src/Tests/When_Subscription_Already_Exists.cs +++ b/src/Tests/if_subscription_already_exists.cs @@ -9,10 +9,10 @@ namespace Tests; // ReSharper disable once InconsistentNaming -public class When_Subscription_Already_Exists(ITestOutputHelper testOutputHelper): DatabaseFixture(testOutputHelper) +public class if_subscription_already_exists(ITestOutputHelper testOutputHelper): DatabaseFixture(testOutputHelper) { [Fact] - public async Task Read_from_transaction_log() + public async Task read_from_transaction_log() { var ct = TimeoutTokenSource().Token; var sharedNamingPolicy = new AttributeNamingPolicy(); diff --git a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs b/src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs similarity index 90% rename from src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs rename to src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs index aad8d93..8509846 100644 --- a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Empty.cs +++ b/src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs @@ -8,10 +8,10 @@ namespace Tests; // ReSharper disable once InconsistentNaming -public class When_Subscription_Does_Not_Exist_And_Table_Is_Empty(ITestOutputHelper testOutputHelper): DatabaseFixture(testOutputHelper) +public class if_subscription_does_not_exist_and_table_is_empty(ITestOutputHelper testOutputHelper): DatabaseFixture(testOutputHelper) { [Fact] - public async Task Read_from_table_using_named_transaction_snapshot() + public async Task read_from_table_using_named_transaction_snapshot() { var ct = TimeoutTokenSource().Token; diff --git a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs b/src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs similarity index 89% rename from src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs rename to src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs index 91ca537..178702c 100644 --- a/src/Tests/When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty.cs +++ b/src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs @@ -8,10 +8,10 @@ namespace Tests; // ReSharper disable once InconsistentNaming -public class When_Subscription_Does_Not_Exist_And_Table_Is_Not_Empty(ITestOutputHelper testOutputHelper): DatabaseFixture(testOutputHelper) +public class if_subscription_does_not_exist_and_table_is_not_empty(ITestOutputHelper testOutputHelper): DatabaseFixture(testOutputHelper) { [Fact] - public async Task Read_from_table_using_named_transaction_snapshot() + public async Task read_from_table_using_named_transaction_snapshot() { var ct = TimeoutTokenSource().Token; var sharedNamingPolicy = new AttributeNamingPolicy(); From a95cf77cfe817c5438dd26a3b9ad3f11485e8bb2 Mon Sep 17 00:00:00 2001 From: giordanol Date: Sun, 21 Jul 2024 12:24:35 +0200 Subject: [PATCH 46/80] table creation can be enforced when bootstrapping in both processes(pub/sub) --- Blumchen.sln | 8 +------- docker-compose.yml | 4 ---- docker/postgres/init.sql | 6 ------ src/Publisher/Program.cs | 20 ++++++++++++-------- src/Publisher/Publisher.csproj | 10 +++++++--- 5 files changed, 20 insertions(+), 28 deletions(-) delete mode 100644 docker/postgres/init.sql diff --git a/Blumchen.sln b/Blumchen.sln index 50edf3c..6186db3 100644 --- a/Blumchen.sln +++ b/Blumchen.sln @@ -36,16 +36,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pgAdmin", "pgAdmin", "{C050 docker\pgAdmin\servers.json = docker\pgAdmin\servers.json EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "postgres", "postgres", "{8AAAA344-B5FD-48D9-B2BA-379E374448D4}" - ProjectSection(SolutionItems) = preProject - docker\postgres\init.sql = docker\postgres\init.sql - EndProjectSection -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "demo", "demo", "{A4044484-FE08-4399-8239-14AABFA30AD7}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SubscriberWorker", "src\SubscriberWorker\SubscriberWorker.csproj", "{DB58DB36-0366-4ABA-BC06-FCA9BB10EB92}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTests", "src\UnitTests\UnitTests.csproj", "{B16305B4-8AC3-4435-AADB-D9E2ACAA1C13}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "UnitTests", "src\UnitTests\UnitTests.csproj", "{B16305B4-8AC3-4435-AADB-D9E2ACAA1C13}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -90,7 +85,6 @@ Global {F2878625-0919-4C26-8DC9-58CD8FA34050} = {A4044484-FE08-4399-8239-14AABFA30AD7} {F81E2D5B-FC59-4396-A911-56BE65E4FE80} = {A4044484-FE08-4399-8239-14AABFA30AD7} {C050E9E8-3FB6-4581-953F-31826E385FB4} = {CD59A1A0-F40D-4047-87A3-66C0F1519FA5} - {8AAAA344-B5FD-48D9-B2BA-379E374448D4} = {CD59A1A0-F40D-4047-87A3-66C0F1519FA5} {DB58DB36-0366-4ABA-BC06-FCA9BB10EB92} = {A4044484-FE08-4399-8239-14AABFA30AD7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/docker-compose.yml b/docker-compose.yml index e34c14e..fa2eee0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,9 +15,6 @@ services: - "wal_level=logical" - "-c" - "wal_compression=on" - volumes: - - ./docker/postgres/init.sql:/docker-entrypoint-initdb.d/init.sql - pgadmin: container_name: pgadmin_container image: dpage/pgadmin4 @@ -36,4 +33,3 @@ services: depends_on: - postgres restart: unless-stopped - diff --git a/docker/postgres/init.sql b/docker/postgres/init.sql deleted file mode 100644 index 5887ecd..0000000 --- a/docker/postgres/init.sql +++ /dev/null @@ -1,6 +0,0 @@ - -CREATE TABLE outbox ( - id BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, - message_type VARCHAR(250) NOT NULL, - data JSONB NOT NULL -); diff --git a/src/Publisher/Program.cs b/src/Publisher/Program.cs index 5278a96..d6aad3f 100644 --- a/src/Publisher/Program.cs +++ b/src/Publisher/Program.cs @@ -2,6 +2,7 @@ using Blumchen.Publisher; using Blumchen.Serialization; using Commons; +using Microsoft.Extensions.Logging; using Npgsql; using Publisher; using UserCreated = Publisher.UserCreated; @@ -13,6 +14,14 @@ Console.WriteLine("How many messages do you want to publish?(press CTRL+C to exit):"); var cts = new CancellationTokenSource(); +var generator = new Func[] +{ + () => new UserCreated(Guid.NewGuid()), + () => new UserDeleted(Guid.NewGuid()), + () => new UserModified(Guid.NewGuid()), + () => new UserSubscribed(Guid.NewGuid()) +}; + do { @@ -29,7 +38,8 @@ //Or you might want to verify at a later stage await new NpgsqlDataSourceBuilder(Settings.ConnectionString) - .Build() + .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())) + .Build() .EnsureTableExists(resolver.TableDescriptor, cts.Token).ConfigureAwait(false); var messages = result / 4; @@ -40,13 +50,7 @@ //use a command for each message { var @events = Enumerable.Range(0, result).Select(i => - (i % 4) switch - { - 0 => new UserCreated(Guid.NewGuid()) as object, - 1 => new UserDeleted(Guid.NewGuid()), - 2 => new UserModified(Guid.NewGuid()), - _ => new UserSubscribed(Guid.NewGuid()) - }); + generator[i % generator.Length]()); await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 0) ? 1 : 0)} {nameof(UserCreated)}"); await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 1) ? 1 : 0)} {nameof(UserDeleted)}"); await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 2) ? 1 : 0)} {nameof(UserModified)}"); diff --git a/src/Publisher/Publisher.csproj b/src/Publisher/Publisher.csproj index 86277ed..0c7153c 100644 --- a/src/Publisher/Publisher.csproj +++ b/src/Publisher/Publisher.csproj @@ -1,4 +1,4 @@ - + net8.0 @@ -9,8 +9,12 @@ - - + + + + + + From d3c5825e5cc650fd8b3b4024093d84e5b314e970 Mon Sep 17 00:00:00 2001 From: giordanol Date: Sun, 21 Jul 2024 13:23:38 +0200 Subject: [PATCH 47/80] typo --- src/Blumchen/Subscriber/OptionsBuilder.cs | 14 +++++++------- src/Subscriber/Program.cs | 4 ++-- src/UnitTests/subscriber_options_builder.cs | 14 +++++++------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index b09cc61..779c9b6 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -101,15 +101,15 @@ public OptionsBuilder Consumes(IMessageHandler handler) where T : class } [UsedImplicitly] - public OptionsBuilder ConsumesRowObject(IMessageHandler handler) where T : class - => ConsumesRow(handler, RawUrnAttribute.RawData.Object, ObjectReplicationDataMapper.Instance); + public OptionsBuilder ConsumesRawObject(IMessageHandler handler) where T : class + => ConsumesRaw(handler, RawUrnAttribute.RawData.Object, ObjectReplicationDataMapper.Instance); [UsedImplicitly] - public OptionsBuilder ConsumesRowString(IMessageHandler handler) where T : class - => ConsumesRow(handler, RawUrnAttribute.RawData.String, StringReplicationDataMapper.Instance); + public OptionsBuilder ConsumesRawString(IMessageHandler handler) where T : class + => ConsumesRaw(handler, RawUrnAttribute.RawData.String, StringReplicationDataMapper.Instance); [UsedImplicitly] - public OptionsBuilder ConsumesRowStrings(IMessageHandler handler) + public OptionsBuilder ConsumesRawStrings(IMessageHandler handler) { _replicationDataMapperSelector.Add(WildCard, new Tuple(StringReplicationDataMapper.Instance, handler)); @@ -117,14 +117,14 @@ public OptionsBuilder ConsumesRowStrings(IMessageHandler handler) } [UsedImplicitly] - public OptionsBuilder ConsumesRowObjects(IMessageHandler handler) + public OptionsBuilder ConsumesRawObjects(IMessageHandler handler) { _replicationDataMapperSelector.Add(WildCard, new Tuple(ObjectReplicationDataMapper.Instance, handler)); return this; } - private OptionsBuilder ConsumesRow(IMessageHandler handler, RawUrnAttribute.RawData filter, + private OptionsBuilder ConsumesRaw(IMessageHandler handler, RawUrnAttribute.RawData filter, IReplicationJsonBMapper dataMapper) where T : class { var urns = typeof(T) diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index 6b22160..bb8c262 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -36,8 +36,8 @@ .NamingPolicy(new AttributeNamingPolicy()) .Consumes(consumer) .JsonContext(SourceGenerationContext.Default) - .ConsumesRowString(consumer) - .ConsumesRowObject(consumer), ct + .ConsumesRawString(consumer) + .ConsumesRawObject(consumer), ct ).GetAsyncEnumerator(ct); await using var cursor1 = cursor.ConfigureAwait(false); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested); diff --git a/src/UnitTests/subscriber_options_builder.cs b/src/UnitTests/subscriber_options_builder.cs index 61df25d..4ad4e80 100644 --- a/src/UnitTests/subscriber_options_builder.cs +++ b/src/UnitTests/subscriber_options_builder.cs @@ -63,7 +63,7 @@ public void requires_at_least_one_method_call_to_consumes() public void has_default_options() { var messageHandler = Substitute.For>(); - var opts = _builder(ValidConnectionString).ConsumesRowStrings(messageHandler).Build(); + var opts = _builder(ValidConnectionString).ConsumesRawStrings(messageHandler).Build(); Assert.NotNull(opts.PublicationOptions); Assert.Equal(CreateStyle.WhenNotExists, opts.PublicationOptions.CreateStyle); @@ -81,18 +81,18 @@ public void has_default_options() } [Fact] - public void with_ConsumesRowStrings() + public void with_ConsumesRawStrings() { var messageHandler = Substitute.For>(); - var opts = _builder(ValidConnectionString).ConsumesRowStrings(messageHandler).Build(); + var opts = _builder(ValidConnectionString).ConsumesRawStrings(messageHandler).Build(); Assert.Equivalent(new Dictionary> { { OptionsBuilder.WildCard, new Tuple(StringReplicationDataMapper.Instance, messageHandler) } }, opts.Registry); } [Fact] - public void with_ConsumesRowObjects() + public void with_ConsumesRawObjects() { var messageHandler = Substitute.For>(); - var opts = _builder(ValidConnectionString).ConsumesRowObjects(messageHandler).Build(); + var opts = _builder(ValidConnectionString).ConsumesRawObjects(messageHandler).Build(); Assert.Equivalent(new Dictionary> { { OptionsBuilder.WildCard, new Tuple(ObjectReplicationDataMapper.Instance, messageHandler) } }, opts.Registry); } @@ -100,7 +100,7 @@ public void with_ConsumesRowObjects() public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() { var messageHandler = Substitute.For>(); - var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRowObject(messageHandler).Build()); + var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawObject(messageHandler).Build()); Assert.IsType(exception); Assert.Equal("No `NamingPolicy` method called on OptionsBuilder", exception.Message); } @@ -109,7 +109,7 @@ public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() public void with_typed_raw_consumer_of_string_requires_RawUrn_decoration() { var messageHandler = Substitute.For>(); - var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRowString(messageHandler).Build()); + var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawString(messageHandler).Build()); Assert.IsType(exception); Assert.Equal("No `NamingPolicy` method called on OptionsBuilder", exception.Message); } From f1d096167bfd0272a5d4084e6f17bc1d2a3d03cd Mon Sep 17 00:00:00 2001 From: giordanol Date: Sun, 21 Jul 2024 14:00:54 +0200 Subject: [PATCH 48/80] Enforcing invariants on publisher/subscriber options builder --- src/Blumchen/{Subscriber => }/Ensure.cs | 5 ++- src/Blumchen/Publisher/OptionsBuilder.cs | 17 ++++++---- src/Blumchen/Subscriber/OptionsBuilder.cs | 17 ++++++---- src/UnitTests/publisher_options_builder.cs | 37 +++++++++++++++++++++ src/UnitTests/subscriber_options_builder.cs | 25 +++++++++++++- 5 files changed, 86 insertions(+), 15 deletions(-) rename src/Blumchen/{Subscriber => }/Ensure.cs (78%) create mode 100644 src/UnitTests/publisher_options_builder.cs diff --git a/src/Blumchen/Subscriber/Ensure.cs b/src/Blumchen/Ensure.cs similarity index 78% rename from src/Blumchen/Subscriber/Ensure.cs rename to src/Blumchen/Ensure.cs index 22edc28..c1ab8ac 100644 --- a/src/Blumchen/Subscriber/Ensure.cs +++ b/src/Blumchen/Ensure.cs @@ -1,12 +1,14 @@ using System.Collections; +using Blumchen.Subscriber; -namespace Blumchen.Subscriber; +namespace Blumchen; internal static class Ensure { public static void Null(T value, params object[] parameters) => new NullTrait().IsValid(value, parameters); public static void NotNull(T value, params object[] parameters) => new NotNullTrait().IsValid(value, parameters); public static void NotEmpty(T value, params object[] parameters) => new NotEmptyTrait().IsValid(value, parameters); + public static void Empty(T value, params object[] parameters) => new EmptyTrait().IsValid(value, parameters); } internal abstract class Validable(Func condition, string errorFormat) @@ -21,3 +23,4 @@ public void IsValid(T value, params object[] parameters) internal class NullTrait(): Validable(v => v is null, $"`{{0}}` method on {nameof(OptionsBuilder)} called more then once"); internal class NotNullTrait(): Validable(v => v is not null, $"`{{0}}` method not called on {nameof(OptionsBuilder)}"); internal class NotEmptyTrait(): Validable(v => v is ICollection { Count: > 0 }, $"No `{{0}}` method called on {nameof(OptionsBuilder)}"); +internal class EmptyTrait(): Validable(v => v is ICollection { Count: 0 }, $"`{{0}}` cannot be mixed with other consuming strategies"); diff --git a/src/Blumchen/Publisher/OptionsBuilder.cs b/src/Blumchen/Publisher/OptionsBuilder.cs index eb7785b..e4d9582 100644 --- a/src/Blumchen/Publisher/OptionsBuilder.cs +++ b/src/Blumchen/Publisher/OptionsBuilder.cs @@ -7,10 +7,15 @@ namespace Blumchen.Publisher; public class OptionsBuilder { - private INamingPolicy? _namingPolicy; - private JsonSerializerContext? _jsonSerializerContext; + [System.Diagnostics.CodeAnalysis.NotNull] + private INamingPolicy? _namingPolicy = default; + + [System.Diagnostics.CodeAnalysis.NotNull] + private JsonSerializerContext? _jsonSerializerContext = default; + private static readonly TableDescriptorBuilder TableDescriptorBuilder = new(); - private MessageTable? _tableDescriptor; + + private MessageTable? _tableDescriptor = default; [UsedImplicitly] public OptionsBuilder NamingPolicy(INamingPolicy namingPolicy) @@ -35,10 +40,10 @@ public OptionsBuilder WithTable(Func builder) { - _messageTable = builder(_tableDescriptorBuilder).Build(); + _tableDescriptor = builder(TableDescriptorBuilder).Build(); return this; } @@ -111,6 +110,8 @@ public OptionsBuilder ConsumesRawString(IMessageHandler handler) wher [UsedImplicitly] public OptionsBuilder ConsumesRawStrings(IMessageHandler handler) { + Ensure.Empty(_replicationDataMapperSelector, nameof(ConsumesRawStrings)); + _replicationDataMapperSelector.Add(WildCard, new Tuple(StringReplicationDataMapper.Instance, handler)); return this; @@ -119,6 +120,8 @@ public OptionsBuilder ConsumesRawStrings(IMessageHandler handler) [UsedImplicitly] public OptionsBuilder ConsumesRawObjects(IMessageHandler handler) { + Ensure.Empty(_replicationDataMapperSelector, nameof(ConsumesRawObjects)); + _replicationDataMapperSelector.Add(WildCard, new Tuple(ObjectReplicationDataMapper.Instance, handler)); return this; @@ -149,7 +152,7 @@ public OptionsBuilder WithErrorProcessor(IErrorProcessor? errorProcessor) internal ISubscriberOptions Build() { - _messageTable ??= _tableDescriptorBuilder.Build(); + _tableDescriptor ??= TableDescriptorBuilder.Build(); Ensure.NotNull(_connectionStringBuilder, $"{nameof(ConnectionString)}"); Ensure.NotNull(_dataSource, $"{nameof(DataSource)}"); @@ -175,12 +178,12 @@ internal ISubscriberOptions Build() throw new ConfigurationException($"`${nameof(Consumes)}<>` requires a valid `{nameof(JsonContext)}`."); } } - + Ensure.NotEmpty(_replicationDataMapperSelector, $"{nameof(Consumes)}..."); _publicationOptions = _publicationOptions with { RegisteredTypes = _replicationDataMapperSelector.Keys.Except([WildCard]).ToHashSet(), - TableDescriptor = _messageTable + TableDescriptor = _tableDescriptor }; return new SubscriberOptions( _dataSource, diff --git a/src/UnitTests/publisher_options_builder.cs b/src/UnitTests/publisher_options_builder.cs new file mode 100644 index 0000000..c80da69 --- /dev/null +++ b/src/UnitTests/publisher_options_builder.cs @@ -0,0 +1,37 @@ +using Blumchen; +using Blumchen.Publisher; +using Blumchen.Serialization; + +namespace UnitTests +{ + public class publisher_options_builder + { + + [Fact] + public void requires_a_method_call_to_JsonContext() + { + var exception = Record.Exception(() => new OptionsBuilder().Build()); + Assert.IsType(exception); + Assert.Equal("`JsonContext` method not called on OptionsBuilder", exception.Message); + } + + [Fact] + public void requires_a_method_call_to_NamingPolicy() + { + var exception = Record.Exception(() => + new OptionsBuilder().JsonContext(SourceGenerationContext.Default).Build()); + Assert.IsType(exception); + Assert.Equal("`NamingPolicy` method not called on OptionsBuilder", exception.Message); + } + + [Fact] + public void has_default_options() + { + var opts = new OptionsBuilder().JsonContext(SourceGenerationContext.Default) + .NamingPolicy(new AttributeNamingPolicy()).Build(); + + Assert.NotNull(opts.JsonTypeResolver); + Assert.Equal(new TableDescriptorBuilder().Build(), opts.TableDescriptor); + } + } +} diff --git a/src/UnitTests/subscriber_options_builder.cs b/src/UnitTests/subscriber_options_builder.cs index 4ad4e80..6f1ee00 100644 --- a/src/UnitTests/subscriber_options_builder.cs +++ b/src/UnitTests/subscriber_options_builder.cs @@ -4,7 +4,6 @@ using Blumchen.Subscriptions.Replication; using Npgsql; using NSubstitute; -using static Blumchen.Subscriptions.Management.PublicationManagement; using static Blumchen.Subscriptions.Subscription; namespace UnitTests @@ -96,6 +95,30 @@ public void with_ConsumesRawObjects() Assert.Equivalent(new Dictionary> { { OptionsBuilder.WildCard, new Tuple(ObjectReplicationDataMapper.Instance, messageHandler) } }, opts.Registry); } + [Fact] + public void ConsumesRawObjects_cannot_be_mixed_with_other_consuming_strategies() + { + var messageHandler1 = Substitute.For>(); + var messageHandler2 = Substitute.For>(); + var exception = Record.Exception(() => + _builder(ValidConnectionString).ConsumesRawStrings(messageHandler2).ConsumesRawObjects(messageHandler1) + .Build()); + Assert.IsType(exception); + Assert.Equal("`ConsumesRawObjects` cannot be mixed with other consuming strategies", exception.Message); + } + + [Fact] + public void ConsumesRawStrings_cannot_be_mixed_with_other_consuming_strategies() + { + var messageHandler1 = Substitute.For>(); + var messageHandler2 = Substitute.For>(); + var exception = Record.Exception(() => + _builder(ValidConnectionString).ConsumesRawObjects(messageHandler1).ConsumesRawStrings(messageHandler2) + .Build()); + Assert.IsType(exception); + Assert.Equal("`ConsumesRawStrings` cannot be mixed with other consuming strategies", exception.Message); + } + [Fact] public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() { From 910902a3162b68c38c03806e0dae17d481dbf751 Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 22 Jul 2024 08:08:04 +0200 Subject: [PATCH 49/80] updated satellite packages to latest --- src/Blumchen/Blumchen.csproj | 4 ++-- src/Subscriber/Subscriber.csproj | 1 + src/SubscriberWorker/SubscriberWorker.csproj | 3 ++- src/Tests/Tests.csproj | 8 ++++---- src/UnitTests/UnitTests.csproj | 6 +++--- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Blumchen/Blumchen.csproj b/src/Blumchen/Blumchen.csproj index d275b6b..9b4e0d9 100644 --- a/src/Blumchen/Blumchen.csproj +++ b/src/Blumchen/Blumchen.csproj @@ -1,4 +1,4 @@ - + 0.1.1 @@ -48,7 +48,7 @@ - + all none all diff --git a/src/Subscriber/Subscriber.csproj b/src/Subscriber/Subscriber.csproj index 7878764..1cc04fd 100644 --- a/src/Subscriber/Subscriber.csproj +++ b/src/Subscriber/Subscriber.csproj @@ -12,6 +12,7 @@ + diff --git a/src/SubscriberWorker/SubscriberWorker.csproj b/src/SubscriberWorker/SubscriberWorker.csproj index ad5c3f6..17aa463 100644 --- a/src/SubscriberWorker/SubscriberWorker.csproj +++ b/src/SubscriberWorker/SubscriberWorker.csproj @@ -11,7 +11,8 @@ - + + diff --git a/src/Tests/Tests.csproj b/src/Tests/Tests.csproj index 029d4bc..342eda0 100644 --- a/src/Tests/Tests.csproj +++ b/src/Tests/Tests.csproj @@ -1,4 +1,4 @@ - + net8.0 @@ -9,12 +9,12 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/UnitTests/UnitTests.csproj b/src/UnitTests/UnitTests.csproj index 96ce6e4..060dba9 100644 --- a/src/UnitTests/UnitTests.csproj +++ b/src/UnitTests/UnitTests.csproj @@ -13,12 +13,12 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive From 5126bb9b034f1f9060cc765df45ca5981c613b8a Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 22 Jul 2024 18:57:32 +0200 Subject: [PATCH 50/80] simplify test --- src/Tests/DatabaseFixture.cs | 4 ++-- src/Tests/if_subscription_already_exists.cs | 2 +- .../if_subscription_does_not_exist_and_table_is_empty.cs | 2 +- .../if_subscription_does_not_exist_and_table_is_not_empty.cs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index 06664c0..61a13db 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -68,7 +68,7 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri await command.ExecuteNonQueryAsync(ct); } - protected (TestMessageHandler handler, OptionsBuilder subscriptionOptionsBuilder) SetupFor( + protected OptionsBuilder SetupFor( string connectionString, string eventsTable, JsonSerializerContext info, @@ -94,7 +94,7 @@ protected static async Task InsertPoisoningMessage(string connectionString, stri .WithReplicationOptions( new ReplicationSlotManagement.ReplicationSlotOptions(slotName ?? Randomise("events_slot")) ); - return (consumer, subscriptionOptionsBuilder); + return subscriptionOptionsBuilder; } private sealed record TestOutErrorProcessor(ITestOutputHelper Output): IErrorProcessor diff --git a/src/Tests/if_subscription_already_exists.cs b/src/Tests/if_subscription_already_exists.cs index 875a487..6e3eb27 100644 --- a/src/Tests/if_subscription_already_exists.cs +++ b/src/Tests/if_subscription_already_exists.cs @@ -29,7 +29,7 @@ public async Task read_from_transaction_log() await dataSource.CreatePublication(publicationName, eventsTable, new HashSet{"urn:message:user-created:v1"}, ct); - var (_, subscriptionOptions) = SetupFor(connectionString, eventsTable, + var subscriptionOptions = SetupFor(connectionString, eventsTable, SubscriberContext.Default, sharedNamingPolicy, Output.WriteLine, publicationName: publicationName, slotName: slotName); //subscriber ignored msg diff --git a/src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs b/src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs index 8509846..346e29a 100644 --- a/src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs +++ b/src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs @@ -34,7 +34,7 @@ public async Task read_from_table_using_named_transaction_snapshot() await MessageAppender.AppendAsync(@event, resolver, connectionString, ct); - var ( _, subscriptionOptions) = SetupFor(connectionString, eventsTable, + var subscriptionOptions = SetupFor(connectionString, eventsTable, SubscriberContext.Default, sharedNamingPolicy, Output.WriteLine); var subscription = new Subscription(); await using var subscription1 = subscription.ConfigureAwait(false); diff --git a/src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs b/src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs index 178702c..9cb2e00 100644 --- a/src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs +++ b/src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs @@ -35,7 +35,7 @@ public async Task read_from_table_using_named_transaction_snapshot() var @expected = new SubscriberUserCreated(@event.Id, @event.Name); - var ( _, subscriptionOptions) = + var subscriptionOptions = SetupFor(connectionString, eventsTable, SubscriberContext.Default, sharedNamingPolicy, Output.WriteLine); var subscription = new Subscription(); await using var subscription1 = subscription.ConfigureAwait(false); From b236c91c1f526c688fce58d56bc1ebf791472948 Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 22 Jul 2024 18:57:56 +0200 Subject: [PATCH 51/80] additional examples --- src/Subscriber/Program.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index bb8c262..0ddf62b 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -33,11 +33,16 @@ .MessageType("message_type") .MessageData("data", new MimeType.Json()) ) - .NamingPolicy(new AttributeNamingPolicy()) - .Consumes(consumer) - .JsonContext(SourceGenerationContext.Default) - .ConsumesRawString(consumer) - .ConsumesRawObject(consumer), ct + .NamingPolicy(new AttributeNamingPolicy()) + .Consumes(consumer) + .JsonContext(SourceGenerationContext.Default) + .ConsumesRawString(consumer) + .ConsumesRawObject(consumer) + //OR + //.ConsumesRawStrings(consumer) + //OR + //.ConsumesRawObjects(consumer) + , ct ).GetAsyncEnumerator(ct); await using var cursor1 = cursor.ConfigureAwait(false); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested); From 49408fa9f555fef48e4fae289987714735230cd7 Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 22 Jul 2024 18:58:55 +0200 Subject: [PATCH 52/80] consumer lookup logic must fallback on willdcard --- src/Blumchen/IDictionaryExtensions.cs | 11 +++++++++ src/Blumchen/Subscriptions/Subscription.cs | 28 +++++++++++++++------- 2 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 src/Blumchen/IDictionaryExtensions.cs diff --git a/src/Blumchen/IDictionaryExtensions.cs b/src/Blumchen/IDictionaryExtensions.cs new file mode 100644 index 0000000..6e3dc9e --- /dev/null +++ b/src/Blumchen/IDictionaryExtensions.cs @@ -0,0 +1,11 @@ +using System.Collections; +using Blumchen.Subscriber; + +namespace Blumchen; + +internal static class IDictionaryExtensions +{ + public static TR FindByMultiKey(this IDictionary registry, params T[] parameters) + where T : class => + !registry.TryGetValue(parameters[0], out var value) ? registry.FindByMultiKey(parameters[1..parameters.Length]) : value; +} diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 2cd5c56..6cbb1a0 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -17,17 +17,27 @@ public sealed class Subscription: IAsyncDisposable private LogicalReplicationConnection? _connection; private readonly OptionsBuilder _builder = new(); - private readonly Func - _messageHandler = Memoizer>, Type, (IMessageHandler messageHandler, MethodInfo methodInfo)> _messageHandler; + + public Subscription() + { + _messageHandler = Memoizer>, Type, (IMessageHandler messageHandler, MethodInfo methodInfo)>.Execute(MessageHandler); + } - private static (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler( - string messageType, IMessageHandler messageHandler, Type objType) + private (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler( + string messageType, IDictionary> registry, Type objType) { - var methodInfos = messageHandler.GetType().GetMethods(BindingFlags.Instance | BindingFlags.Public); - var methodInfo = methodInfos.SingleOrDefault(mi => mi.GetParameters().Any(pa => pa.ParameterType == objType)) - ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); - return (messageHandler, methodInfo); + var tuple = registry.FindByMultiKey(messageType, OptionsBuilder.WildCard) ?? + throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); + { + var messageHandler = tuple.Item2; + var methodInfos = messageHandler.GetType().GetMethods(BindingFlags.Instance | BindingFlags.Public); + var methodInfo = + methodInfos.SingleOrDefault(mi => mi.GetParameters().Any(pa => pa.ParameterType == objType)) + ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); + return (messageHandler, methodInfo); + } } public enum CreateStyle @@ -121,7 +131,7 @@ IErrorProcessor errorProcessor case OkEnvelope(var value, var messageType): { var (messageHandler, methodInfo) = - _messageHandler(messageType, registry[messageType].Item2, value.GetType()); + _messageHandler(messageType, registry, value.GetType()); await ((Task)methodInfo.Invoke(messageHandler, [value])!).ConfigureAwait(false); yield return envelope; From 376d7489d0bc1065f5dde7416049b9346b54d2e8 Mon Sep 17 00:00:00 2001 From: giordanol Date: Tue, 23 Jul 2024 09:29:52 +0200 Subject: [PATCH 53/80] use select pg_advisory_xact_lock to serialize access to message table creation --- src/Blumchen/Database/Run.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Blumchen/Database/Run.cs b/src/Blumchen/Database/Run.cs index eca94a5..6ffc326 100644 --- a/src/Blumchen/Database/Run.cs +++ b/src/Blumchen/Database/Run.cs @@ -13,11 +13,11 @@ private static async Task Execute( CancellationToken ct) { await using var command = dataSource.CreateCommand(sql); - await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + await command.ExecuteNonQueryAsync(ct); } public static async Task EnsureTableExists(this NpgsqlDataSource dataSource, TableDescriptorBuilder.MessageTable tableDescriptor, CancellationToken ct) - => await dataSource.Execute(tableDescriptor.ToString(), ct).ConfigureAwait(false); + => await dataSource.Execute(string.Concat("select pg_advisory_xact_lock(12345);", tableDescriptor), ct).ConfigureAwait(false); public static async Task Exists( this NpgsqlDataSource dataSource, From 8794c4f1c9ffc9ce03cd766d4aa07ce872059723 Mon Sep 17 00:00:00 2001 From: giordanol Date: Tue, 23 Jul 2024 10:28:01 +0200 Subject: [PATCH 54/80] mime type is internally exposed for future extension towards binary data format, actually not used --- src/Blumchen/Subscriptions/MimeType.cs | 5 ++++- src/Blumchen/TableDescriptorBuilder.cs | 6 +++--- src/Subscriber/Program.cs | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Blumchen/Subscriptions/MimeType.cs b/src/Blumchen/Subscriptions/MimeType.cs index 1197908..4874cdd 100644 --- a/src/Blumchen/Subscriptions/MimeType.cs +++ b/src/Blumchen/Subscriptions/MimeType.cs @@ -2,5 +2,8 @@ namespace Blumchen.Subscriptions; public abstract record MimeType(string mimeType) { - public record Json(): MimeType("application/json"); + internal record JsonMimeType(): MimeType("application/json"); + + public static MimeType Json => new JsonMimeType(); + } diff --git a/src/Blumchen/TableDescriptorBuilder.cs b/src/Blumchen/TableDescriptorBuilder.cs index ec91b9b..ab16c1a 100644 --- a/src/Blumchen/TableDescriptorBuilder.cs +++ b/src/Blumchen/TableDescriptorBuilder.cs @@ -22,9 +22,9 @@ public TableDescriptorBuilder Id(string name) return this; } - public TableDescriptorBuilder MessageData(string name, MimeType mime) + public TableDescriptorBuilder MessageData(string name) { - TableDescriptor = TableDescriptor with { Data = new Column.Data(name), MimeType = mime }; + TableDescriptor = TableDescriptor with { Data = new Column.Data(name), MimeType = MimeType.Json }; return this; } @@ -43,7 +43,7 @@ public record MessageTable(string Name = MessageTable.DefaultName) public Column.Id Id { get; internal init; } = Column.Id.Default(); public Column.MessageType MessageType { get; internal init; } = Column.MessageType.Default(); public Column.Data Data { get; internal init; } = Column.Data.Default(); - public MimeType MimeType { get; internal init; } = new MimeType.Json(); + public MimeType MimeType { get; internal init; } = MimeType.Json; public MessageTable Build() => this; public override string ToString() => @$" diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index 0ddf62b..590e522 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -31,7 +31,7 @@ .WithTable(options => options .Id("id") .MessageType("message_type") - .MessageData("data", new MimeType.Json()) + .MessageData("data") ) .NamingPolicy(new AttributeNamingPolicy()) .Consumes(consumer) From fa055881006d4f6f043fd4cabea37abd0552e539 Mon Sep 17 00:00:00 2001 From: giordanol Date: Tue, 23 Jul 2024 10:30:44 +0200 Subject: [PATCH 55/80] renamed nethod Name => Named for table name --- src/Blumchen/TableDescriptorBuilder.cs | 11 ++++++----- src/Tests/DatabaseFixture.cs | 4 ++-- src/Tests/if_subscription_already_exists.cs | 2 +- ..._subscription_does_not_exist_and_table_is_empty.cs | 2 +- ...scription_does_not_exist_and_table_is_not_empty.cs | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Blumchen/TableDescriptorBuilder.cs b/src/Blumchen/TableDescriptorBuilder.cs index ab16c1a..4a2147e 100644 --- a/src/Blumchen/TableDescriptorBuilder.cs +++ b/src/Blumchen/TableDescriptorBuilder.cs @@ -10,7 +10,7 @@ public record TableDescriptorBuilder public MessageTable Build() => TableDescriptor.Build(); - public TableDescriptorBuilder Name(string eventsTable) + public TableDescriptorBuilder Named(string eventsTable) { TableDescriptor = new MessageTable(eventsTable); return this; @@ -46,12 +46,13 @@ public record MessageTable(string Name = MessageTable.DefaultName) public MimeType MimeType { get; internal init; } = MimeType.Json; public MessageTable Build() => this; - public override string ToString() => @$" + public override string ToString() => $""" CREATE TABLE IF NOT EXISTS {Name} ( - {Id} PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, + {Id} PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, {MessageType} NOT NULL, - {Data} NOT NULL - );"; + {Data} NOT NULL + ); + """; } public record Column(string Name, NpgsqlDbType Type) diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index 61a13db..a1c9c94 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -49,7 +49,7 @@ CancellationToken ct { var tableName = Randomise("outbox"); - var tableDesc = new TableDescriptorBuilder().Name(tableName).Build(); + var tableDesc = new TableDescriptorBuilder().Named(tableName).Build(); await dataSource.EnsureTableExists(tableDesc, ct).ConfigureAwait(false); return tableName; @@ -87,7 +87,7 @@ protected OptionsBuilder SetupFor( .JsonContext(info) .NamingPolicy(namingPolicy) .Consumes(consumer) - .WithTable(o => o.Name(eventsTable)) + .WithTable(o => o.Named(eventsTable)) .WithPublicationOptions( new PublicationManagement.PublicationOptions(PublicationName: publicationName ?? Randomise("events_pub")) ) diff --git a/src/Tests/if_subscription_already_exists.cs b/src/Tests/if_subscription_already_exists.cs index 6e3eb27..19c2a24 100644 --- a/src/Tests/if_subscription_already_exists.cs +++ b/src/Tests/if_subscription_already_exists.cs @@ -22,7 +22,7 @@ public async Task read_from_transaction_log() var opts = new OptionsBuilder() .JsonContext(PublisherContext.Default) .NamingPolicy(sharedNamingPolicy) - .WithTable(o => o.Name(eventsTable)) + .WithTable(o => o.Named(eventsTable)) .Build(); var slotName = "subscription_test"; var publicationName = "publication_test"; diff --git a/src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs b/src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs index 346e29a..2f74ecd 100644 --- a/src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs +++ b/src/Tests/if_subscription_does_not_exist_and_table_is_empty.cs @@ -21,7 +21,7 @@ public async Task read_from_table_using_named_transaction_snapshot() var resolver = new OptionsBuilder() .JsonContext(PublisherContext.Default) .NamingPolicy(sharedNamingPolicy) - .WithTable(o => o.Name(eventsTable)) + .WithTable(o => o.Named(eventsTable)) .Build(); //subscriber ignored msg await MessageAppender.AppendAsync(new PublisherUserDeleted(Guid.NewGuid(), Guid.NewGuid().ToString()), resolver, connectionString, ct); diff --git a/src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs b/src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs index 9cb2e00..7a720de 100644 --- a/src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs +++ b/src/Tests/if_subscription_does_not_exist_and_table_is_not_empty.cs @@ -21,7 +21,7 @@ public async Task read_from_table_using_named_transaction_snapshot() var resolver = new OptionsBuilder() .JsonContext(PublisherContext.Default) .NamingPolicy(sharedNamingPolicy) - .WithTable(o => o.Name(eventsTable)) + .WithTable(o => o.Named(eventsTable)) .Build(); //subscriber ignored msg From 72054462d8199629cc6dd61e38b26b14a8812212 Mon Sep 17 00:00:00 2001 From: giordanol Date: Tue, 23 Jul 2024 10:32:30 +0200 Subject: [PATCH 56/80] tested table creation --- src/Blumchen/IDictionaryExtensions.cs | 3 -- src/UnitTests/UnitTests.csproj | 1 + src/UnitTests/message_table_creation.cs | 51 +++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 src/UnitTests/message_table_creation.cs diff --git a/src/Blumchen/IDictionaryExtensions.cs b/src/Blumchen/IDictionaryExtensions.cs index 6e3dc9e..48fe3f3 100644 --- a/src/Blumchen/IDictionaryExtensions.cs +++ b/src/Blumchen/IDictionaryExtensions.cs @@ -1,6 +1,3 @@ -using System.Collections; -using Blumchen.Subscriber; - namespace Blumchen; internal static class IDictionaryExtensions diff --git a/src/UnitTests/UnitTests.csproj b/src/UnitTests/UnitTests.csproj index 060dba9..17ace07 100644 --- a/src/UnitTests/UnitTests.csproj +++ b/src/UnitTests/UnitTests.csproj @@ -7,6 +7,7 @@ + diff --git a/src/UnitTests/message_table_creation.cs b/src/UnitTests/message_table_creation.cs new file mode 100644 index 0000000..378ac84 --- /dev/null +++ b/src/UnitTests/message_table_creation.cs @@ -0,0 +1,51 @@ +using Blumchen; +using FsCheck.Xunit; + +namespace UnitTests +{ + public class message_table_creation + { + [Fact] + public void default_table_descriptor() + { + const string sql = """ + CREATE TABLE IF NOT EXISTS outbox ( + id Bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, + message_type Varchar(250) NOT NULL, + data Jsonb NOT NULL + ); + """; + + var implicitTableDescriptor = new TableDescriptorBuilder().Build(); + var explicitTableDescriptor = new TableDescriptorBuilder().UseDefaults().Build(); + Assert.Equal(implicitTableDescriptor.ToString(), explicitTableDescriptor.ToString()); + Assert.Equal(sql, implicitTableDescriptor.ToString()); + } + + [Property] + public void with_varying_descriptor( + string tableName, + string idColName, + string messageTypeColName, + int messageTypeColDimension, + string dataColName) + { + var sql = $""" + CREATE TABLE IF NOT EXISTS {tableName} ( + {idColName} Bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, + {messageTypeColName} Varchar({messageTypeColDimension}) NOT NULL, + {dataColName} Jsonb NOT NULL + ); + """; + + var tableDescriptor = new TableDescriptorBuilder() + .Named(tableName) + .Id(idColName) + .MessageType(messageTypeColName, messageTypeColDimension) + .MessageData(dataColName) + .Build(); + + Assert.Equal(sql, tableDescriptor.ToString()); + } + } +} From ddbd7f6c386b937b3ba91b42342c4132129ad80c Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 24 Jul 2024 10:19:51 +0200 Subject: [PATCH 57/80] simplified raw urn --- src/Blumchen/Ensure.cs | 3 ++ .../Serialization/MessageUrnAttribute.cs | 39 +------------------ src/Blumchen/Subscriber/OptionsBuilder.cs | 9 ++--- src/Subscriber/Contracts.cs | 4 +- src/UnitTests/Contracts.cs | 4 +- src/UnitTests/subscriber_options_builder.cs | 4 +- 6 files changed, 14 insertions(+), 49 deletions(-) diff --git a/src/Blumchen/Ensure.cs b/src/Blumchen/Ensure.cs index c1ab8ac..07cbaee 100644 --- a/src/Blumchen/Ensure.cs +++ b/src/Blumchen/Ensure.cs @@ -1,10 +1,12 @@ using System.Collections; +using Blumchen.Serialization; using Blumchen.Subscriber; namespace Blumchen; internal static class Ensure { + public static void RawUrn(T value, params object[] parameters) => new RawUrnTrait().IsValid(value, parameters); public static void Null(T value, params object[] parameters) => new NullTrait().IsValid(value, parameters); public static void NotNull(T value, params object[] parameters) => new NotNullTrait().IsValid(value, parameters); public static void NotEmpty(T value, params object[] parameters) => new NotEmptyTrait().IsValid(value, parameters); @@ -20,6 +22,7 @@ public void IsValid(T value, params object[] parameters) } } +internal class RawUrnTrait(): Validable(v => v is ICollection { Count: > 0 }, $"`{nameof(RawUrnAttribute)}` missing on `{typeof(TR).Name}` message type"); internal class NullTrait(): Validable(v => v is null, $"`{{0}}` method on {nameof(OptionsBuilder)} called more then once"); internal class NotNullTrait(): Validable(v => v is not null, $"`{{0}}` method not called on {nameof(OptionsBuilder)}"); internal class NotEmptyTrait(): Validable(v => v is ICollection { Count: > 0 }, $"No `{{0}}` method called on {nameof(OptionsBuilder)}"); diff --git a/src/Blumchen/Serialization/MessageUrnAttribute.cs b/src/Blumchen/Serialization/MessageUrnAttribute.cs index ac2b4c5..dfc2db7 100644 --- a/src/Blumchen/Serialization/MessageUrnAttribute.cs +++ b/src/Blumchen/Serialization/MessageUrnAttribute.cs @@ -32,45 +32,8 @@ private static Uri FormatUrn(string urn) } } - [AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = false)] -public class RawUrnAttribute: Attribute -{ - public enum RawData - { - String, - Object - } - - public RawData Data { get; } - - /// - /// - /// The urn value to use for this message type. - /// The value to bind to this - public RawUrnAttribute(string urn, RawData data) - { - Data = data; - ArgumentException.ThrowIfNullOrEmpty(urn, nameof(urn)); - - if (urn.StartsWith(MessageUrn.Prefix)) - throw new ArgumentException($"Value should not contain the default prefix '{MessageUrn.Prefix}'.", nameof(urn)); - - Urn = FormatUrn(urn); - } - - public Uri Urn { get; } - - private static Uri FormatUrn(string urn) - { - var fullValue = MessageUrn.Prefix + urn; - - if (Uri.TryCreate(fullValue, UriKind.Absolute, out var uri)) - return uri; - - throw new UriFormatException($"Invalid URN: {fullValue}"); - } -} +public class RawUrnAttribute(string urn): MessageUrnAttribute(urn); public static class MessageUrn { diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index 8ffdedb..9c7fb5a 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -101,11 +101,11 @@ public OptionsBuilder Consumes(IMessageHandler handler) where T : class [UsedImplicitly] public OptionsBuilder ConsumesRawObject(IMessageHandler handler) where T : class - => ConsumesRaw(handler, RawUrnAttribute.RawData.Object, ObjectReplicationDataMapper.Instance); + => ConsumesRaw(handler, ObjectReplicationDataMapper.Instance); [UsedImplicitly] public OptionsBuilder ConsumesRawString(IMessageHandler handler) where T : class - => ConsumesRaw(handler, RawUrnAttribute.RawData.String, StringReplicationDataMapper.Instance); + => ConsumesRaw(handler, StringReplicationDataMapper.Instance); [UsedImplicitly] public OptionsBuilder ConsumesRawStrings(IMessageHandler handler) @@ -127,15 +127,14 @@ public OptionsBuilder ConsumesRawObjects(IMessageHandler handler) return this; } - private OptionsBuilder ConsumesRaw(IMessageHandler handler, RawUrnAttribute.RawData filter, + private OptionsBuilder ConsumesRaw(IMessageHandler handler, IReplicationJsonBMapper dataMapper) where T : class { var urns = typeof(T) .GetCustomAttributes(typeof(RawUrnAttribute), false) .OfType() - .Where(attribute => attribute.Data == filter) .Select(attribute => attribute.Urn).ToList(); - Ensure.NotEmpty>(urns, nameof(NamingPolicy)); + Ensure.RawUrn,T>(urns, nameof(NamingPolicy)); using var urnEnum = urns.GetEnumerator(); while (urnEnum.MoveNext()) _replicationDataMapperSelector.Add(urnEnum.Current.ToString(), diff --git a/src/Subscriber/Contracts.cs b/src/Subscriber/Contracts.cs index 7fb0a57..19e99ba 100644 --- a/src/Subscriber/Contracts.cs +++ b/src/Subscriber/Contracts.cs @@ -9,11 +9,11 @@ public record UserCreatedContract( string Name ); - [RawUrn("user-deleted:v1", RawUrnAttribute.RawData.Object)] + [RawUrn("user-deleted:v1")] public class MessageObjects; - [RawUrn("user-modified:v1", RawUrnAttribute.RawData.String)] + [RawUrn("user-modified:v1")] internal class MessageString; [JsonSourceGenerationOptions(WriteIndented = true)] diff --git a/src/UnitTests/Contracts.cs b/src/UnitTests/Contracts.cs index 8e7191a..faba680 100644 --- a/src/UnitTests/Contracts.cs +++ b/src/UnitTests/Contracts.cs @@ -9,11 +9,11 @@ public record UserCreatedContract( string Name ); - [RawUrn("user-deleted:v1", RawUrnAttribute.RawData.Object)] + [RawUrn("user-deleted:v1")] public class MessageObjects; - [RawUrn("user-modified:v1", RawUrnAttribute.RawData.String)] + [RawUrn("user-modified:v1")] internal class MessageString; internal class InvalidMessage; diff --git a/src/UnitTests/subscriber_options_builder.cs b/src/UnitTests/subscriber_options_builder.cs index 6f1ee00..21f33a6 100644 --- a/src/UnitTests/subscriber_options_builder.cs +++ b/src/UnitTests/subscriber_options_builder.cs @@ -125,7 +125,7 @@ public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() var messageHandler = Substitute.For>(); var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawObject(messageHandler).Build()); Assert.IsType(exception); - Assert.Equal("No `NamingPolicy` method called on OptionsBuilder", exception.Message); + Assert.Equal("`RawUrnAttribute` missing on `InvalidMessage` message type", exception.Message); } [Fact] @@ -134,7 +134,7 @@ public void with_typed_raw_consumer_of_string_requires_RawUrn_decoration() var messageHandler = Substitute.For>(); var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawString(messageHandler).Build()); Assert.IsType(exception); - Assert.Equal("No `NamingPolicy` method called on OptionsBuilder", exception.Message); + Assert.Equal("`RawUrnAttribute` missing on `InvalidMessage` message type", exception.Message); } } } From 964b88353e693f74b168f4d370c833b1354eb335 Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 24 Jul 2024 11:26:48 +0200 Subject: [PATCH 58/80] Provide untyped append method --- src/Blumchen/Publisher/MessageAppender.cs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/Blumchen/Publisher/MessageAppender.cs b/src/Blumchen/Publisher/MessageAppender.cs index da7d43c..bdb8ccb 100644 --- a/src/Blumchen/Publisher/MessageAppender.cs +++ b/src/Blumchen/Publisher/MessageAppender.cs @@ -7,6 +7,26 @@ namespace Blumchen.Publisher; public static class MessageAppender { + public static async Task AppendAsync(object @input + , PublisherOptions resolver + , NpgsqlConnection connection + , NpgsqlTransaction transaction + , CancellationToken ct + ) + { + switch (@input) + { + case null: + throw new ArgumentNullException(nameof(@input)); + case IEnumerable inputs: + await AppendBatchAsyncOfT(inputs, resolver.TableDescriptor, resolver.JsonTypeResolver, connection, transaction, ct).ConfigureAwait(false); + break; + default: + await AppendAsyncOfT(input, resolver.TableDescriptor, resolver.JsonTypeResolver, connection, transaction, ct).ConfigureAwait(false); + break; + } + } + public static async Task AppendAsync(T @input , PublisherOptions resolver , NpgsqlConnection connection @@ -34,7 +54,7 @@ private static async Task AppendAsyncOfT(T input , NpgsqlTransaction transaction , CancellationToken ct) where T : class { - var (typeName, jsonTypeInfo) = typeResolver.Resolve(typeof(T)); + var (typeName, jsonTypeInfo) = typeResolver.Resolve(input.GetType()); var data = JsonSerialization.ToJson(@input, jsonTypeInfo); await using var command = new NpgsqlCommand( From 1de0a77e5d505f472fa8dcae8c4a5041cf1816bd Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 24 Jul 2024 11:28:26 +0200 Subject: [PATCH 59/80] avoid usage checking on public method --- src/Blumchen/Publisher/PublisherOptions.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Blumchen/Publisher/PublisherOptions.cs b/src/Blumchen/Publisher/PublisherOptions.cs index 48ace93..711fc03 100644 --- a/src/Blumchen/Publisher/PublisherOptions.cs +++ b/src/Blumchen/Publisher/PublisherOptions.cs @@ -1,6 +1,7 @@ using System.Text.Json.Serialization.Metadata; using Blumchen.Database; using Blumchen.Serialization; +using JetBrains.Annotations; using Npgsql; namespace Blumchen.Publisher; @@ -9,13 +10,15 @@ public record PublisherOptions(TableDescriptorBuilder.MessageTable TableDescript public static class PublisherOptionsExtensions { + [UsedImplicitly] public static async Task EnsureTable(this PublisherOptions publisherOptions, NpgsqlDataSource dataSource, CancellationToken ct) { await dataSource.EnsureTableExists(publisherOptions.TableDescriptor, ct); return publisherOptions; } - public static Task EnsureTable(this PublisherOptions publisherOptions, + [UsedImplicitly] + public static async Task EnsureTable(this PublisherOptions publisherOptions, string connectionString, CancellationToken ct) - => EnsureTable(publisherOptions, new NpgsqlDataSourceBuilder(connectionString).Build(), ct); + => await EnsureTable(publisherOptions, new NpgsqlDataSourceBuilder(connectionString).Build(), ct); } From 575c2b567e414365588c065fbe4ced0b05fb05c5 Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 24 Jul 2024 11:29:48 +0200 Subject: [PATCH 60/80] provide additional usage patterns --- src/Publisher/Program.cs | 93 +++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/src/Publisher/Program.cs b/src/Publisher/Program.cs index d6aad3f..6497d0d 100644 --- a/src/Publisher/Program.cs +++ b/src/Publisher/Program.cs @@ -11,70 +11,73 @@ using UserSubscribed = Publisher.UserSubscribed; Console.Title = typeof(Program).Assembly.GetName().Name!; -Console.WriteLine("How many messages do you want to publish?(press CTRL+C to exit):"); -var cts = new CancellationTokenSource(); -var generator = new Func[] +var generator = new Dictionary> { - () => new UserCreated(Guid.NewGuid()), - () => new UserDeleted(Guid.NewGuid()), - () => new UserModified(Guid.NewGuid()), - () => new UserSubscribed(Guid.NewGuid()) + { nameof(UserCreated), () => new UserCreated(Guid.NewGuid()) }, + { nameof(UserDeleted), () => new UserDeleted(Guid.NewGuid()) }, + { nameof(UserModified), () => new UserModified(Guid.NewGuid()) }, + { nameof(UserSubscribed), () => new UserSubscribed(Guid.NewGuid()) } }; +var cts = new CancellationTokenSource(); + +var resolver = await new OptionsBuilder() + .JsonContext(SourceGenerationContext.Default) + .NamingPolicy(new AttributeNamingPolicy()) + .WithTable(builder => builder.UseDefaults()) //default, but explicit + .Build() + .EnsureTable(Settings.ConnectionString, cts.Token)//enforce table existence and conformity - db roundtrip + .ConfigureAwait(false); + +//Or you might want to verify at a later stage +await new NpgsqlDataSourceBuilder(Settings.ConnectionString) + .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())) + .Build() + .EnsureTableExists(resolver.TableDescriptor, cts.Token).ConfigureAwait(false); + do { - + await Console.Out.WriteLineAsync("How many messages do you want to publish?(press CTRL+C to exit):"); var line = Console.ReadLine(); if (line != null && int.TryParse(line, out var result)) { - var resolver = await new OptionsBuilder() - .JsonContext(SourceGenerationContext.Default) - .NamingPolicy(new AttributeNamingPolicy()) - .WithTable(builder => builder.UseDefaults()) //default, but explicit - .Build() - .EnsureTable(Settings.ConnectionString, cts.Token)//enforce table existence and conformity - db roundtrip - .ConfigureAwait(false); - - //Or you might want to verify at a later stage - await new NpgsqlDataSourceBuilder(Settings.ConnectionString) - .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())) - .Build() - .EnsureTableExists(resolver.TableDescriptor, cts.Token).ConfigureAwait(false); - - var messages = result / 4; + var generatorLength = generator.Count; + var messageCount = result / generatorLength; var ct = cts.Token; var connection = new NpgsqlConnection(Settings.ConnectionString); await using var connection1 = connection.ConfigureAwait(false); await connection.OpenAsync(ct).ConfigureAwait(false); //use a command for each message { - var @events = Enumerable.Range(0, result).Select(i => - generator[i % generator.Length]()); - await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 0) ? 1 : 0)} {nameof(UserCreated)}"); - await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 1) ? 1 : 0)} {nameof(UserDeleted)}"); - await Console.Out.WriteLineAsync($"Publishing {messages + ((result % 3 > 2) ? 1 : 0)} {nameof(UserModified)}"); - await Console.Out.WriteLineAsync($"Publishing {messages} {nameof(UserSubscribed)}"); - foreach (var @event in @events) + var tuple = Enumerable.Range(0, result).Select(i => + generator.ElementAt(i % generatorLength)); + + foreach (var s in generator.Keys.Select((key, i) => $"Publishing {(messageCount + (result % generatorLength > i ? 1 : 0))} {key}").ToList()) + await Console.Out.WriteLineAsync(s); + + foreach (var message in tuple.Select(_ => _.Value())) { var transaction = await connection.BeginTransactionAsync(ct).ConfigureAwait(false); try { - switch (@event) - { - case UserCreated m: - await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); - break; - case UserDeleted m: - await MessageAppender.AppendAsync( m, resolver, connection, transaction, ct).ConfigureAwait(false); - break; - case UserModified m: - await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); - break; - case UserSubscribed m: - await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); - break; - } + await MessageAppender.AppendAsync(message, resolver, connection, transaction, ct).ConfigureAwait(false); + //OR with typed version + //switch (message) + //{ + // case UserCreated m: + // await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); + // break; + // case UserDeleted m: + // await MessageAppender.AppendAsync( m, resolver, connection, transaction, ct).ConfigureAwait(false); + // break; + // case UserModified m: + // await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); + // break; + // case UserSubscribed m: + // await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); + // break; + //} await transaction.CommitAsync(ct).ConfigureAwait(false); } From 44953bb372b7f4b1be335b8cd7ff74138887ab55 Mon Sep 17 00:00:00 2001 From: giordanol Date: Sat, 27 Jul 2024 14:14:47 +0200 Subject: [PATCH 61/80] provide minimal dsl on typed consumer --- src/Blumchen/Ensure.cs | 15 +++-- .../Subscriber/OptionsBuilder.Consumes.cs | 57 +++++++++++++++++++ src/Blumchen/Subscriber/OptionsBuilder.cs | 33 ++--------- src/Subscriber/Program.cs | 4 +- src/SubscriberWorker/Program.cs | 10 ++-- src/Tests/DatabaseFixture.cs | 4 +- 6 files changed, 83 insertions(+), 40 deletions(-) create mode 100644 src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs diff --git a/src/Blumchen/Ensure.cs b/src/Blumchen/Ensure.cs index 07cbaee..4cfeb05 100644 --- a/src/Blumchen/Ensure.cs +++ b/src/Blumchen/Ensure.cs @@ -6,19 +6,22 @@ namespace Blumchen; internal static class Ensure { - public static void RawUrn(T value, params object[] parameters) => new RawUrnTrait().IsValid(value, parameters); - public static void Null(T value, params object[] parameters) => new NullTrait().IsValid(value, parameters); - public static void NotNull(T value, params object[] parameters) => new NotNullTrait().IsValid(value, parameters); - public static void NotEmpty(T value, params object[] parameters) => new NotEmptyTrait().IsValid(value, parameters); - public static void Empty(T value, params object[] parameters) => new EmptyTrait().IsValid(value, parameters); + public static void RawUrn(T value, string parameters) => new RawUrnTrait().IsValid(value, parameters); + public static void Null(T value, string parameters) => new NullTrait().IsValid(value, parameters); + public static void NotNull(T value, string parameters) => new NotNullTrait().IsValid(value, parameters); + public static void NotEmpty(T value, string parameters) => new NotEmptyTrait().IsValid(value, parameters); + public static void Empty(T value, string parameters) => new EmptyTrait().IsValid(value, parameters); + public static bool Empty(T value1, TU value2, params string[] parameters) => + new EmptyTrait().IsValid(value1, parameters) && new EmptyTrait().IsValid(value2, parameters); } internal abstract class Validable(Func condition, string errorFormat) { - public void IsValid(T value, params object[] parameters) + public bool IsValid(T value, params string[] parameters) { if (!condition(value)) throw new ConfigurationException(string.Format(errorFormat, parameters)); + return true; } } diff --git a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs new file mode 100644 index 0000000..83a6ad8 --- /dev/null +++ b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs @@ -0,0 +1,57 @@ +using System.Text.Json.Serialization; +using Blumchen.Serialization; +using Blumchen.Subscriptions.Replication; +using JetBrains.Annotations; + +namespace Blumchen.Subscriber; + +public sealed partial class OptionsBuilder +{ + private INamingPolicy? _namingPolicy; + private JsonSerializerContext? _jsonSerializerContext; + + [UsedImplicitly] + internal OptionsBuilder NamingPolicy(INamingPolicy namingPolicy) + { + _namingPolicy = namingPolicy; + return this; + } + + public interface INamingOptionsContext + { + OptionsBuilder NamingPolicy(INamingPolicy namingPolicy); + } + + internal class NamingOptionsContext(OptionsBuilder builder): INamingOptionsContext + { + public OptionsBuilder NamingPolicy(INamingPolicy namingPolicy) + => builder.NamingPolicy(namingPolicy); + } + + public interface IConsumesTypedJsonOptionsContext + { + INamingOptionsContext JsonContext(JsonSerializerContext jsonSerializerContext); + IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) where T : class; + } + + internal class ConsumesTypedJsonTypedJsonOptionsContext(OptionsBuilder builder): IConsumesTypedJsonOptionsContext + { + public INamingOptionsContext JsonContext(JsonSerializerContext jsonSerializerContext) + { + builder._jsonSerializerContext = jsonSerializerContext; + return new NamingOptionsContext(builder); + } + + public IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) where T : class + { + return builder.Consumes(handler); + } + } + + public IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) where T : class + { + Ensure.Empty(_replicationDataMapperSelector, nameof(Consumes)); + _typeRegistry.Add(typeof(T), handler); + return new ConsumesTypedJsonTypedJsonOptionsContext(this); + } +} diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index 9c7fb5a..d2de044 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -8,7 +8,7 @@ namespace Blumchen.Subscriber; -public sealed class OptionsBuilder +public sealed partial class OptionsBuilder { internal const string WildCard = "*"; @@ -26,7 +26,6 @@ private readonly Dictionary(IMessageHandler handler) where T : class - { - _typeRegistry.Add(typeof(T), handler); - return this; - } - [UsedImplicitly] public OptionsBuilder ConsumesRawObject(IMessageHandler handler) where T : class => ConsumesRaw(handler, ObjectReplicationDataMapper.Instance); @@ -110,8 +87,8 @@ public OptionsBuilder ConsumesRawString(IMessageHandler handler) wher [UsedImplicitly] public OptionsBuilder ConsumesRawStrings(IMessageHandler handler) { - Ensure.Empty(_replicationDataMapperSelector, nameof(ConsumesRawStrings)); - + Ensure.Empty(_replicationDataMapperSelector, _typeRegistry, nameof(ConsumesRawStrings)); + _replicationDataMapperSelector.Add(WildCard, new Tuple(StringReplicationDataMapper.Instance, handler)); return this; @@ -120,7 +97,7 @@ public OptionsBuilder ConsumesRawStrings(IMessageHandler handler) [UsedImplicitly] public OptionsBuilder ConsumesRawObjects(IMessageHandler handler) { - Ensure.Empty(_replicationDataMapperSelector, nameof(ConsumesRawObjects)); + Ensure.Empty(_replicationDataMapperSelector, _typeRegistry, nameof(ConsumesRawObjects)); _replicationDataMapperSelector.Add(WildCard, new Tuple(ObjectReplicationDataMapper.Instance, handler)); @@ -174,7 +151,7 @@ internal ISubscriberOptions Build() } else { - throw new ConfigurationException($"`${nameof(Consumes)}<>` requires a valid `{nameof(JsonContext)}`."); + throw new ConfigurationException($"`${nameof(Consumes)}<>` requires a valid `{nameof(JsonSerializerContext)}`."); } } Ensure.NotEmpty(_replicationDataMapperSelector, $"{nameof(Consumes)}..."); diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index 590e522..d71975c 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -33,9 +33,11 @@ .MessageType("message_type") .MessageData("data") ) - .NamingPolicy(new AttributeNamingPolicy()) + .Consumes(consumer) .JsonContext(SourceGenerationContext.Default) + .NamingPolicy(new AttributeNamingPolicy()) + .ConsumesRawString(consumer) .ConsumesRawObject(consumer) //OR diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index 6d3b91b..5e1a226 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -66,10 +66,11 @@ .WithReplicationOptions(new ReplicationSlotManagement.ReplicationSlotOptions($"{nameof(HandleImpl1)}_slot")) .WithPublicationOptions(new PublicationManagement.PublicationOptions($"{nameof(HandleImpl1)}_pub")) .WithErrorProcessor(provider.GetRequiredService()) - .NamingPolicy(provider.GetRequiredService()) - .JsonContext(SourceGenerationContext.Default) + .Consumes(provider.GetRequiredService>()) .Consumes(provider.GetRequiredService>()) + .JsonContext(SourceGenerationContext.Default) + .NamingPolicy(provider.GetRequiredService()) ) .ResiliencyPipeline(provider.GetRequiredService>().GetPipeline("default")) ) @@ -81,9 +82,10 @@ .WithReplicationOptions(new ReplicationSlotManagement.ReplicationSlotOptions($"{nameof(HandleImpl2)}_slot")) .WithPublicationOptions(new PublicationManagement.PublicationOptions($"{nameof(HandleImpl2)}_pub")) .WithErrorProcessor(provider.GetRequiredService()) - .NamingPolicy(provider.GetRequiredService()) - .JsonContext(SourceGenerationContext.Default) + .Consumes(provider.GetRequiredService>()) + .JsonContext(SourceGenerationContext.Default) + .NamingPolicy(provider.GetRequiredService()) ) .ResiliencyPipeline(provider.GetRequiredService>().GetPipeline("default")) ); diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index a1c9c94..c1cba88 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -84,9 +84,11 @@ protected OptionsBuilder SetupFor( .WithErrorProcessor(new TestOutErrorProcessor(Output)) .DataSource(new NpgsqlDataSourceBuilder(connectionString).Build()) .ConnectionString(connectionString) + + .Consumes(consumer) .JsonContext(info) .NamingPolicy(namingPolicy) - .Consumes(consumer) + .WithTable(o => o.Named(eventsTable)) .WithPublicationOptions( new PublicationManagement.PublicationOptions(PublicationName: publicationName ?? Randomise("events_pub")) From 2726be9c4d7a2310df7a7de5b63cc7f0ea513f8f Mon Sep 17 00:00:00 2001 From: giordanol Date: Sat, 27 Jul 2024 14:15:56 +0200 Subject: [PATCH 62/80] move methodInfo registration at configuration time --- src/Blumchen/Subscriber/ISubscriberOptions.cs | 7 +- .../Subscriber/OptionsBuilder.Consumes.cs | 9 ++- src/Blumchen/Subscriber/OptionsBuilder.cs | 45 ++++++++---- .../Replication/IReplicationDataMapper.cs | 12 ++-- .../Replication/IReplicationJsonBMapper.cs | 1 - src/Blumchen/Subscriptions/Subscription.cs | 44 ++++++------ src/Subscriber/Program.cs | 69 +++++++++++++------ src/UnitTests/Contracts.cs | 2 +- src/UnitTests/subscriber_options_builder.cs | 8 +-- 9 files changed, 121 insertions(+), 76 deletions(-) diff --git a/src/Blumchen/Subscriber/ISubscriberOptions.cs b/src/Blumchen/Subscriber/ISubscriberOptions.cs index 3c0e3f6..7ebc706 100644 --- a/src/Blumchen/Subscriber/ISubscriberOptions.cs +++ b/src/Blumchen/Subscriber/ISubscriberOptions.cs @@ -1,3 +1,4 @@ +using System.Reflection; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Replication; using JetBrains.Annotations; @@ -11,7 +12,7 @@ public interface ISubscriberOptions { [UsedImplicitly] NpgsqlDataSource DataSource { get; } [UsedImplicitly] NpgsqlConnectionStringBuilder ConnectionStringBuilder { get; } - IDictionary> Registry { get; } + IDictionary> Registry { get; } [UsedImplicitly] PublicationOptions PublicationOptions { get; } [UsedImplicitly] ReplicationSlotOptions ReplicationOptions { get; } [UsedImplicitly] IErrorProcessor ErrorProcessor { get; } @@ -22,7 +23,7 @@ void Deconstruct( out PublicationOptions publicationOptions, out ReplicationSlotOptions replicationSlotOptions, out IErrorProcessor errorProcessor, - out IDictionary> registry); + out IDictionary> registry); } internal record SubscriberOptions( @@ -31,4 +32,4 @@ internal record SubscriberOptions( PublicationOptions PublicationOptions, ReplicationSlotOptions ReplicationOptions, IErrorProcessor ErrorProcessor, - IDictionary> Registry): ISubscriberOptions; + IDictionary> Registry): ISubscriberOptions; diff --git a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs index 83a6ad8..297f49e 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs @@ -1,3 +1,4 @@ +using System.Reflection; using System.Text.Json.Serialization; using Blumchen.Serialization; using Blumchen.Subscriptions.Replication; @@ -51,7 +52,13 @@ public IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) public IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) where T : class { Ensure.Empty(_replicationDataMapperSelector, nameof(Consumes)); - _typeRegistry.Add(typeof(T), handler); + var methodInfo = handler + .GetType() + .GetMethod(nameof(IMessageHandler.Handle), BindingFlags.Instance | BindingFlags.Public, [typeof(T)]) + ?? throw new ConfigurationException($"Unable to find {nameof(IMessageHandler)} implementation on {handler.GetType().Name}"); + + + _typeRegistry.Add(typeof(T), new Tuple(handler, methodInfo)); return new ConsumesTypedJsonTypedJsonOptionsContext(this); } } diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index d2de044..a83e141 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -1,3 +1,4 @@ +using System.Reflection; using System.Text.Json.Serialization; using Blumchen.Serialization; using Blumchen.Subscriptions; @@ -20,18 +21,15 @@ public sealed partial class OptionsBuilder private PublicationManagement.PublicationOptions _publicationOptions = new(); private ReplicationSlotManagement.ReplicationSlotOptions? _replicationSlotOptions; - private readonly Dictionary _typeRegistry = []; + private readonly Dictionary> _typeRegistry = []; - private readonly Dictionary> + private readonly Dictionary> _replicationDataMapperSelector = []; private IErrorProcessor? _errorProcessor; private static readonly TableDescriptorBuilder TableDescriptorBuilder = new(); private TableDescriptorBuilder.MessageTable? _tableDescriptor; - private readonly IReplicationJsonBMapper _objectDataMapper = - new ObjectReplicationDataMapper(new ObjectReplicationDataReader()); - private IReplicationJsonBMapper? _jsonDataMapper; @@ -77,20 +75,25 @@ public OptionsBuilder WithReplicationOptions( } [UsedImplicitly] - public OptionsBuilder ConsumesRawObject(IMessageHandler handler) where T : class - => ConsumesRaw(handler, ObjectReplicationDataMapper.Instance); + public OptionsBuilder ConsumesRawObject(IMessageHandler handler) where T : class + => ConsumesRaw(handler, ObjectReplicationDataMapper.Instance); [UsedImplicitly] - public OptionsBuilder ConsumesRawString(IMessageHandler handler) where T : class - => ConsumesRaw(handler, StringReplicationDataMapper.Instance); + public OptionsBuilder ConsumesRawString(IMessageHandler handler) where T : class + => ConsumesRaw(handler, StringReplicationDataMapper.Instance); [UsedImplicitly] public OptionsBuilder ConsumesRawStrings(IMessageHandler handler) { Ensure.Empty(_replicationDataMapperSelector, _typeRegistry, nameof(ConsumesRawStrings)); + var methodInfo = handler + .GetType() + .GetMethod(nameof(IMessageHandler.Handle),BindingFlags.Instance | BindingFlags.Public, [typeof(string)]) + ?? throw new ConfigurationException($"Unable to find {nameof(IMessageHandler)} implementation on {handler.GetType().Name}"); + _replicationDataMapperSelector.Add(WildCard, - new Tuple(StringReplicationDataMapper.Instance, handler)); + new Tuple(StringReplicationDataMapper.Instance, handler, methodInfo)); return this; } @@ -99,23 +102,35 @@ public OptionsBuilder ConsumesRawObjects(IMessageHandler handler) { Ensure.Empty(_replicationDataMapperSelector, _typeRegistry, nameof(ConsumesRawObjects)); + var methodInfo = handler + .GetType() + .GetMethod(nameof(IMessageHandler.Handle), BindingFlags.Instance | BindingFlags.Public, [typeof(string)]) + ?? throw new ConfigurationException($"Unable to find {nameof(IMessageHandler)} implementation on {handler.GetType().Name}"); + + _replicationDataMapperSelector.Add(WildCard, - new Tuple(ObjectReplicationDataMapper.Instance, handler)); + new Tuple(ObjectReplicationDataMapper.Instance, handler, methodInfo)); return this; } - private OptionsBuilder ConsumesRaw(IMessageHandler handler, - IReplicationJsonBMapper dataMapper) where T : class + private OptionsBuilder ConsumesRaw(IMessageHandler handler, + IReplicationJsonBMapper dataMapper) where T : class where TU : class { var urns = typeof(T) .GetCustomAttributes(typeof(RawUrnAttribute), false) .OfType() .Select(attribute => attribute.Urn).ToList(); Ensure.RawUrn,T>(urns, nameof(NamingPolicy)); + + var methodInfo = handler + .GetType() + .GetMethod(nameof(IMessageHandler.Handle), BindingFlags.Instance | BindingFlags.Public, [typeof(TU)]) + ?? throw new ConfigurationException($"Unable to find {nameof(IMessageHandler)} implementation on {handler.GetType().Name}"); + using var urnEnum = urns.GetEnumerator(); while (urnEnum.MoveNext()) _replicationDataMapperSelector.Add(urnEnum.Current.ToString(), - new Tuple(dataMapper, handler)); + new Tuple(dataMapper, handler, methodInfo)); return this; } @@ -147,7 +162,7 @@ internal ISubscriberOptions Build() foreach (var (key, value) in typeResolver.RegisteredTypes.Join(_typeRegistry, pair => pair.Value, pair => pair.Key, (pair, valuePair) => (pair.Key, valuePair.Value))) _replicationDataMapperSelector.Add(key, - new Tuple(_jsonDataMapper, value)); + new Tuple(_jsonDataMapper, value.Item1, value.Item2)); } else { diff --git a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs index d248e01..dfcdf3f 100644 --- a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs @@ -1,3 +1,4 @@ +using System.Reflection; using Npgsql; using Npgsql.Replication.PgOutput; using Npgsql.Replication.PgOutput.Messages; @@ -13,14 +14,14 @@ public interface IReplicationDataMapper Task ReadFromReplication(InsertMessage insertMessage, CancellationToken ct); } -internal class ReplicationDataMapper(IDictionary> mapperSelector) +internal class ReplicationDataMapper(IDictionary> mapperSelector) : IReplicationDataMapper { - private readonly Func>, IReplicationJsonBMapper> _memoizer = Memoizer>, IReplicationJsonBMapper>.Execute(SelectMapper); + private readonly Func>, IReplicationJsonBMapper> _memoizer = Memoizer>, IReplicationJsonBMapper>.Execute(SelectMapper); private static IReplicationJsonBMapper SelectMapper(string key, - IDictionary> registry) - => registry.TryGetValue(key, out var tuple) + IDictionary> registry) => + registry.TryGetValue(key, out var tuple) ? tuple.Item1 : registry.TryGetValue(OptionsBuilder.WildCard, out tuple) ? tuple.Item1 @@ -72,8 +73,7 @@ public async Task ReadFromSnapshot(NpgsqlDataReader reader, Cancellat { id = reader.GetInt64(0); var typeName = reader.GetString(1); - - return await mapperSelector[typeName].Item1.ReadFromSnapshot(typeName, id, reader, ct); + return await _memoizer(typeName, mapperSelector).ReadFromSnapshot(typeName, id, reader, ct); } catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or JsonException) { diff --git a/src/Blumchen/Subscriptions/Replication/IReplicationJsonBMapper.cs b/src/Blumchen/Subscriptions/Replication/IReplicationJsonBMapper.cs index 89deb4c..dd267c4 100644 --- a/src/Blumchen/Subscriptions/Replication/IReplicationJsonBMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/IReplicationJsonBMapper.cs @@ -41,7 +41,6 @@ public async Task ReadFromSnapshot(string typeName, long id, NpgsqlDa try { var eventType = resolver?.Resolve(typeName); - ArgumentNullException.ThrowIfNull(eventType, typeName); var value = await replicationDataReader.Read(reader, ct, eventType).ConfigureAwait(false) ?? throw new ArgumentNullException(); return new OkEnvelope(value, typeName); } diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 6cbb1a0..000e631 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -17,27 +17,26 @@ public sealed class Subscription: IAsyncDisposable private LogicalReplicationConnection? _connection; private readonly OptionsBuilder _builder = new(); - private readonly Func>, Type, (IMessageHandler messageHandler, MethodInfo methodInfo)> _messageHandler; + private readonly + Func>, Type, ( + IMessageHandler messageHandler, MethodInfo methodInfo)> _messageHandler; public Subscription() { - _messageHandler = Memoizer>, Type, (IMessageHandler messageHandler, - MethodInfo methodInfo)>.Execute(MessageHandler); + _messageHandler = + Memoizer>, Type, ( + IMessageHandler messageHandler, + MethodInfo methodInfo)>.Execute(MessageHandler); } private (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler( - string messageType, IDictionary> registry, Type objType) + string messageType, IDictionary> registry, + Type objType) { - var tuple = registry.FindByMultiKey(messageType, OptionsBuilder.WildCard) ?? - throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); - { - var messageHandler = tuple.Item2; - var methodInfos = messageHandler.GetType().GetMethods(BindingFlags.Instance | BindingFlags.Public); - var methodInfo = - methodInfos.SingleOrDefault(mi => mi.GetParameters().Any(pa => pa.ParameterType == objType)) - ?? throw new NotSupportedException($"Unregistered type for {objType.AssemblyQualifiedName}"); - return (messageHandler, methodInfo); - } + var (_, messageHandler, methodInfo) = registry.FindByMultiKey(messageType, OptionsBuilder.WildCard) ?? + throw new NotSupportedException( + $"Unregistered type for {objType.AssemblyQualifiedName}"); + return (messageHandler, methodInfo); } public enum CreateStyle @@ -119,7 +118,7 @@ internal async IAsyncEnumerable Subscribe( private async IAsyncEnumerable ProcessEnvelope( IEnvelope envelope, - IDictionary> registry, + IDictionary> registry, IErrorProcessor errorProcessor ) { @@ -129,14 +128,13 @@ IErrorProcessor errorProcessor await errorProcessor.Process(error.Error).ConfigureAwait(false); yield break; case OkEnvelope(var value, var messageType): - { - var (messageHandler, methodInfo) = - _messageHandler(messageType, registry, value.GetType()); - await ((Task)methodInfo.Invoke(messageHandler, [value])!).ConfigureAwait(false); - - yield return envelope; - yield break; - } + { + var (messageHandler, methodInfo) = + _messageHandler(messageType, registry, value.GetType()); + await ((Task)methodInfo.Invoke(messageHandler, [value])!).ConfigureAwait(false); + yield return envelope; + yield break; + } } } diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index d71975c..ec5a72b 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -16,30 +16,40 @@ TaskScheduler.UnobservedTaskException += (_,e) => Console.Out.WriteLine(e.Exception.ToString()); var ct = cancellationTokenSource.Token; -var consumer = new Consumer(); var subscription = new Subscription(); await using var subscription1 = subscription.ConfigureAwait(false); try { + + var loggerFactory = LoggerFactory.Create(builder => builder + .AddFilter("Microsoft", LogLevel.Warning) + .AddFilter("System", LogLevel.Warning) + .AddFilter("Npgsql", LogLevel.Information) + .AddFilter("Blumchen", LogLevel.Debug) + .AddFilter("Subscriber", LogLevel.Trace) + .AddSimpleConsole()); + var logger = loggerFactory.CreateLogger(); var dataSourceBuilder = new NpgsqlDataSourceBuilder(Settings.ConnectionString) - .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())); + .UseLoggerFactory(loggerFactory); var cursor = subscription.Subscribe( - builder => builder - .DataSource(dataSourceBuilder.Build()) - .ConnectionString(Settings.ConnectionString) - .WithTable(options => options - .Id("id") - .MessageType("message_type") - .MessageData("data") - ) - - .Consumes(consumer) - .JsonContext(SourceGenerationContext.Default) - .NamingPolicy(new AttributeNamingPolicy()) - - .ConsumesRawString(consumer) - .ConsumesRawObject(consumer) + builder => + { + var consumer = new Consumer(loggerFactory.CreateLogger()); + return builder + .DataSource(dataSourceBuilder.Build()) + .ConnectionString(Settings.ConnectionString) + .WithTable(options => options + .Id("id") + .MessageType("message_type") + .MessageData("data") + ) + .Consumes(consumer) + .JsonContext(SourceGenerationContext.Default) + .NamingPolicy(new AttributeNamingPolicy()) + .ConsumesRawString(consumer) + .ConsumesRawObject(consumer); + } //OR //.ConsumesRawStrings(consumer) //OR @@ -47,7 +57,9 @@ , ct ).GetAsyncEnumerator(ct); await using var cursor1 = cursor.ConfigureAwait(false); - while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested); + while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested) + if(logger.IsEnabled(LogLevel.Trace)) + logger.LogTrace(cursor.Current.ToString()); } catch (Exception e) { @@ -58,14 +70,27 @@ namespace Subscriber { - internal class Consumer: + internal class Consumer(ILogger logger): IMessageHandler, IMessageHandler, IMessageHandler { - public Task Handle(string value) => Console.Out.WriteLineAsync(value); - public Task Handle(object value) => Console.Out.WriteLineAsync(value.ToString()); - public Task Handle(UserCreatedContract value) => Console.Out.WriteLineAsync(JsonSerialization.ToJson(value, SourceGenerationContext.Default.UserCreatedContract)); + private int _completed; + + private Task ReportSuccess(int count) + { + if (logger.IsEnabled(LogLevel.Debug)) + logger.LogDebug($"Read #{count} messages {typeof(T).FullName}"); + return Task.CompletedTask; + } + + private Task Handle(T value) => + ReportSuccess(Interlocked.Increment(ref _completed)); + + public Task Handle(string value) => Handle(value); + public Task Handle(object value) => Handle(value); + public Task Handle(UserCreatedContract value) => + Handle(value); } } diff --git a/src/UnitTests/Contracts.cs b/src/UnitTests/Contracts.cs index faba680..cdeb01b 100644 --- a/src/UnitTests/Contracts.cs +++ b/src/UnitTests/Contracts.cs @@ -16,7 +16,7 @@ public class MessageObjects; [RawUrn("user-modified:v1")] internal class MessageString; - internal class InvalidMessage; + public class InvalidMessage; [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(UserCreatedContract))] diff --git a/src/UnitTests/subscriber_options_builder.cs b/src/UnitTests/subscriber_options_builder.cs index 21f33a6..b7cb0dc 100644 --- a/src/UnitTests/subscriber_options_builder.cs +++ b/src/UnitTests/subscriber_options_builder.cs @@ -122,8 +122,8 @@ public void ConsumesRawStrings_cannot_be_mixed_with_other_consuming_strategies() [Fact] public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() { - var messageHandler = Substitute.For>(); - var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawObject(messageHandler).Build()); + var messageHandler = Substitute.For>(); + var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawObject(messageHandler).Build()); Assert.IsType(exception); Assert.Equal("`RawUrnAttribute` missing on `InvalidMessage` message type", exception.Message); } @@ -131,8 +131,8 @@ public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() [Fact] public void with_typed_raw_consumer_of_string_requires_RawUrn_decoration() { - var messageHandler = Substitute.For>(); - var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawString(messageHandler).Build()); + var messageHandler = Substitute.For>(); + var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawString(messageHandler).Build()); Assert.IsType(exception); Assert.Equal("`RawUrnAttribute` missing on `InvalidMessage` message type", exception.Message); } From c777721e13926e63af2e94c91373a67eafcc12bd Mon Sep 17 00:00:00 2001 From: giordanol Date: Sat, 27 Jul 2024 14:55:00 +0200 Subject: [PATCH 63/80] enable processed data trace only on trace enabled logging level --- src/Blumchen/DependencyInjection/Worker.cs | 10 ++++++++-- src/Subscriber/Program.cs | 4 ++-- src/SubscriberWorker/Program.cs | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Blumchen/DependencyInjection/Worker.cs b/src/Blumchen/DependencyInjection/Worker.cs index e828089..bd5cba5 100644 --- a/src/Blumchen/DependencyInjection/Worker.cs +++ b/src/Blumchen/DependencyInjection/Worker.cs @@ -14,14 +14,20 @@ public class Worker( private static readonly ConcurrentDictionary> LoggingActions = new(StringComparer.OrdinalIgnoreCase); private static void Notify(ILogger logger, LogLevel level, string template, params object[] parameters) { + LoggingActions.GetOrAdd(template,_ => LoggerAction(level, logger.IsEnabled(level)))(logger, template, parameters); + return; + static Action LoggerAction(LogLevel ll, bool enabled) => (ll, enabled) switch { (LogLevel.Information, true) => (logger, template, parameters) => logger.LogInformation(template, parameters), (LogLevel.Debug, true) => (logger, template, parameters) => logger.LogDebug(template, parameters), + (LogLevel.Trace, true) => (logger, template, parameters) => logger.LogTrace(template, parameters), + (LogLevel.Warning, true) => (logger, template, parameters) => logger.LogWarning(template, parameters), + (LogLevel.Error, true) => (logger, template, parameters) => logger.LogError(template, parameters), + (LogLevel.Critical, true) => (logger, template, parameters) => logger.LogCritical(template, parameters), (_, _) => (_, _, _) => { } }; - LoggingActions.GetOrAdd(template,_ => LoggerAction(level, logger.IsEnabled(level)))(logger, template, parameters); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -33,7 +39,7 @@ await options.ResiliencePipeline.ExecuteAsync(async token => .GetAsyncEnumerator(token); Notify(logger, LogLevel.Information,"{WorkerName} started", WorkerName); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !token.IsCancellationRequested) - Notify(logger, LogLevel.Debug, "{cursor.Current} processed", cursor.Current); + Notify(logger, LogLevel.Trace, "{cursor.Current} processed", cursor.Current); }, stoppingToken).ConfigureAwait(false); Notify(logger, LogLevel.Information, "{WorkerName} stopped", WorkerName); diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index ec5a72b..590931f 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -29,7 +29,7 @@ .AddFilter("Blumchen", LogLevel.Debug) .AddFilter("Subscriber", LogLevel.Trace) .AddSimpleConsole()); - var logger = loggerFactory.CreateLogger(); + var logger = loggerFactory.CreateLogger("Subscriber"); var dataSourceBuilder = new NpgsqlDataSourceBuilder(Settings.ConnectionString) .UseLoggerFactory(loggerFactory); var cursor = subscription.Subscribe( @@ -59,7 +59,7 @@ await using var cursor1 = cursor.ConfigureAwait(false); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested) if(logger.IsEnabled(LogLevel.Trace)) - logger.LogTrace(cursor.Current.ToString()); + logger.LogTrace($"{cursor.Current} processed"); } catch (Exception e) { diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index 5e1a226..129844e 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -49,7 +49,7 @@ .AddFilter("Microsoft", LogLevel.Warning) .AddFilter("System", LogLevel.Warning) .AddFilter("Npgsql", LogLevel.Information) - .AddFilter("Blumchen", LogLevel.Debug) + .AddFilter("Blumchen", LogLevel.Trace) .AddFilter("SubscriberWorker", LogLevel.Debug) .AddSimpleConsole(); }) From b3d4d17ad02d0aee2029ee38ecc2cb4a886fbf7c Mon Sep 17 00:00:00 2001 From: giordanol Date: Tue, 30 Jul 2024 09:47:28 +0200 Subject: [PATCH 64/80] corrected IErrorProcessor signature to accept KoEnvelope Id --- src/Blumchen/Subscriptions/IErrorProcessor.cs | 4 ++-- src/Blumchen/Subscriptions/Subscription.cs | 2 +- src/Tests/DatabaseFixture.cs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Blumchen/Subscriptions/IErrorProcessor.cs b/src/Blumchen/Subscriptions/IErrorProcessor.cs index b3cec9d..527c579 100644 --- a/src/Blumchen/Subscriptions/IErrorProcessor.cs +++ b/src/Blumchen/Subscriptions/IErrorProcessor.cs @@ -2,10 +2,10 @@ namespace Blumchen.Subscriptions; public interface IErrorProcessor { - Func Process { get; } + Func Process { get; } } public record ConsoleOutErrorProcessor: IErrorProcessor { - public Func Process => exception => Console.Out.WriteLineAsync($"record id:{0} resulted in error:{exception.Message}"); + public Func Process => (exception, id) => Console.Out.WriteLineAsync($"record id:{id} resulted in error:{exception.Message}"); } diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 000e631..4b5a43d 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -125,7 +125,7 @@ IErrorProcessor errorProcessor switch (envelope) { case KoEnvelope error: - await errorProcessor.Process(error.Error).ConfigureAwait(false); + await errorProcessor.Process(error.Error, error.Id).ConfigureAwait(false); yield break; case OkEnvelope(var value, var messageType): { diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index c1cba88..b6ad753 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -101,9 +101,9 @@ protected OptionsBuilder SetupFor( private sealed record TestOutErrorProcessor(ITestOutputHelper Output): IErrorProcessor { - public Func Process => exception => + public Func Process => (exception, id) => { - Output.WriteLine($"record id:{0} resulted in error:{exception.Message}"); + Output.WriteLine($"record id:{id} resulted in error:{exception.Message}"); return Task.CompletedTask; }; } From 8fd8041317bbfab839d1c6a7c134603eaa866b20 Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 31 Jul 2024 18:54:17 +0200 Subject: [PATCH 65/80] added more publishing options --- src/Publisher/Program.cs | 170 +++++++++++++++++++-------------- src/Publisher/Publisher.csproj | 1 + 2 files changed, 101 insertions(+), 70 deletions(-) diff --git a/src/Publisher/Program.cs b/src/Publisher/Program.cs index 6497d0d..c0d8e7b 100644 --- a/src/Publisher/Program.cs +++ b/src/Publisher/Program.cs @@ -1,6 +1,8 @@ using Blumchen.Database; using Blumchen.Publisher; using Blumchen.Serialization; +using CommandLine; +using CommandLine.Text; using Commons; using Microsoft.Extensions.Logging; using Npgsql; @@ -12,13 +14,7 @@ Console.Title = typeof(Program).Assembly.GetName().Name!; -var generator = new Dictionary> -{ - { nameof(UserCreated), () => new UserCreated(Guid.NewGuid()) }, - { nameof(UserDeleted), () => new UserDeleted(Guid.NewGuid()) }, - { nameof(UserModified), () => new UserModified(Guid.NewGuid()) }, - { nameof(UserSubscribed), () => new UserSubscribed(Guid.NewGuid()) } -}; +var generator = Options.Generator; var cts = new CancellationTokenSource(); @@ -35,75 +31,109 @@ .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())) .Build() .EnsureTableExists(resolver.TableDescriptor, cts.Token).ConfigureAwait(false); - +Parser.Default.ParseArguments("--help".Split()); do { - await Console.Out.WriteLineAsync("How many messages do you want to publish?(press CTRL+C to exit):"); - var line = Console.ReadLine(); - if (line != null && int.TryParse(line, out var result)) + async void GenerateFn(Options o) => await Generate(generator.Join(o.MessageTypes, pair => pair.Key, s => s, (pair, s) => pair).ToDictionary(), o.Count, resolver, cts); + + Parser.Default.ParseArguments(Console.ReadLine()?.Split()) + .WithParsed(GenerateFn) + .WithNotParsed(HandleParseError); + +} while (true); + +static void HandleParseError(IEnumerable errs) => Console.WriteLine("Errors:" + string.Join(',', errs.Select(e => e.Tag))); + +async Task Generate(Dictionary> dictionary, int result, PublisherOptions publisherOptions, + CancellationTokenSource cancellationTokenSource) +{ + var generatorLength = dictionary.Count; + var messageCount = result / generatorLength; + var ct = cancellationTokenSource.Token; + var connection = new NpgsqlConnection(Settings.ConnectionString); + await using var connection1 = connection.ConfigureAwait(false); + await connection.OpenAsync(ct).ConfigureAwait(false); + //use a command for each message { - var generatorLength = generator.Count; - var messageCount = result / generatorLength; - var ct = cts.Token; - var connection = new NpgsqlConnection(Settings.ConnectionString); - await using var connection1 = connection.ConfigureAwait(false); - await connection.OpenAsync(ct).ConfigureAwait(false); - //use a command for each message - { - var tuple = Enumerable.Range(0, result).Select(i => - generator.ElementAt(i % generatorLength)); + var tuple = Enumerable.Range(0, result).Select(i => + dictionary.ElementAt(i % generatorLength)); + + foreach (var s in dictionary.Keys.Select((key, i) => $"Publishing {(messageCount + (result % generatorLength > i ? 1 : 0))} {key}").ToList()) + await Console.Out.WriteLineAsync(s); - foreach (var s in generator.Keys.Select((key, i) => $"Publishing {(messageCount + (result % generatorLength > i ? 1 : 0))} {key}").ToList()) - await Console.Out.WriteLineAsync(s); + foreach (var message in tuple.Select(pair => pair.Value())) + { + var transaction = await connection.BeginTransactionAsync(ct).ConfigureAwait(false); + try + { + await MessageAppender.AppendAsync(message, publisherOptions, connection, transaction, ct).ConfigureAwait(false); + //OR with typed version + //switch (message) + //{ + // case UserCreated m: + // await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); + // break; + // case UserDeleted m: + // await MessageAppender.AppendAsync( m, resolver, connection, transaction, ct).ConfigureAwait(false); + // break; + // case UserModified m: + // await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); + // break; + // case UserSubscribed m: + // await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); + // break; + //} - foreach (var message in tuple.Select(_ => _.Value())) + await transaction.CommitAsync(ct).ConfigureAwait(false); + } + catch (Exception e) { - var transaction = await connection.BeginTransactionAsync(ct).ConfigureAwait(false); - try - { - await MessageAppender.AppendAsync(message, resolver, connection, transaction, ct).ConfigureAwait(false); - //OR with typed version - //switch (message) - //{ - // case UserCreated m: - // await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); - // break; - // case UserDeleted m: - // await MessageAppender.AppendAsync( m, resolver, connection, transaction, ct).ConfigureAwait(false); - // break; - // case UserModified m: - // await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); - // break; - // case UserSubscribed m: - // await MessageAppender.AppendAsync(m, resolver, connection, transaction, ct).ConfigureAwait(false); - // break; - //} - - await transaction.CommitAsync(ct).ConfigureAwait(false); - } - catch (Exception e) - { - await transaction.RollbackAsync(ct).ConfigureAwait(false); - Console.WriteLine(e); - throw; - } + await transaction.RollbackAsync(ct).ConfigureAwait(false); + Console.WriteLine(e); + throw; } - await Console.Out.WriteLineAsync($"Published {result} messages!"); } - //use a batch command - //{ - // var transaction = await connection.BeginTransactionAsync(ct); - // try - // { - // var @events = Enumerable.Range(0, result) - // .Select(i1 => new UserCreated(Guid.NewGuid(), Guid.NewGuid().ToString())); - // await MessageAppender.AppendAsync(@events, resolver, connection, transaction, ct); - // } - // catch (Exception e) - // { - // Console.WriteLine(e); - // throw; - // } - //} + await Console.Out.WriteLineAsync($"Published {result} messages!"); } -} while (true); + //use a batch command + //{ + // var transaction = await connection.BeginTransactionAsync(ct); + // try + // { + // var @events = Enumerable.Range(0, result) + // .Select(i1 => new UserCreated(Guid.NewGuid(), Guid.NewGuid().ToString())); + // await MessageAppender.AppendAsync(@events, resolver, connection, transaction, ct); + // } + // catch (Exception e) + // { + // Console.WriteLine(e); + // throw; + // } + //} +} + +internal class Options(IEnumerable messageTypes, int count) +{ + internal static readonly Dictionary> Generator = new() + { + { nameof(UserCreated), () => new UserCreated(Guid.NewGuid()) }, + { nameof(UserDeleted), () => new UserDeleted(Guid.NewGuid()) }, + { nameof(UserModified), () => new UserModified(Guid.NewGuid()) }, + { nameof(UserSubscribed), () => new UserSubscribed(Guid.NewGuid()) } + }; + private static readonly int CountByType = TotalCount / Generator.Count; + private static readonly int Mod = TotalCount % CountByType; + private const int TotalCount = 10; + + [Option('t', "type", Required = true, HelpText = "Message type.", Separator = '|')] + public IEnumerable MessageTypes { get; } = messageTypes; + + [Option('c', "count", Required = true, HelpText = "Total number")] + public int Count { get; } = count; + + [Usage] + public static IEnumerable Examples => + [ + new Example($"Publish {string.Join(" and ", Generator.Keys.Select((type,i)=> $"{CountByType + (Mod>i?1:0)} {type}"))} messages", new Options(Generator.Keys, TotalCount)) + ]; +} diff --git a/src/Publisher/Publisher.csproj b/src/Publisher/Publisher.csproj index 0c7153c..e6a52a5 100644 --- a/src/Publisher/Publisher.csproj +++ b/src/Publisher/Publisher.csproj @@ -9,6 +9,7 @@ + From e56c3c17b1ce616cade4ebc4196f103c715ca3db Mon Sep 17 00:00:00 2001 From: giordanol Date: Wed, 31 Jul 2024 18:56:21 +0200 Subject: [PATCH 66/80] embed ConsumeOptions with typed Consumes --- .../Subscriber/OptionsBuilder.Consumes.cs | 13 ++++--- src/Subscriber/Program.cs | 7 ++-- src/SubscriberWorker/Program.cs | 38 +++++++++++-------- src/Tests/DatabaseFixture.cs | 9 +++-- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs index 297f49e..c37179b 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs @@ -20,24 +20,24 @@ internal OptionsBuilder NamingPolicy(INamingPolicy namingPolicy) public interface INamingOptionsContext { - OptionsBuilder NamingPolicy(INamingPolicy namingPolicy); + OptionsBuilder AndNamingPolicy(INamingPolicy namingPolicy); } internal class NamingOptionsContext(OptionsBuilder builder): INamingOptionsContext { - public OptionsBuilder NamingPolicy(INamingPolicy namingPolicy) + public OptionsBuilder AndNamingPolicy(INamingPolicy namingPolicy) => builder.NamingPolicy(namingPolicy); } public interface IConsumesTypedJsonOptionsContext { - INamingOptionsContext JsonContext(JsonSerializerContext jsonSerializerContext); + INamingOptionsContext WithJsonContext(JsonSerializerContext jsonSerializerContext); IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) where T : class; } internal class ConsumesTypedJsonTypedJsonOptionsContext(OptionsBuilder builder): IConsumesTypedJsonOptionsContext { - public INamingOptionsContext JsonContext(JsonSerializerContext jsonSerializerContext) + public INamingOptionsContext WithJsonContext(JsonSerializerContext jsonSerializerContext) { builder._jsonSerializerContext = jsonSerializerContext; return new NamingOptionsContext(builder); @@ -49,7 +49,7 @@ public IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) } } - public IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) where T : class + internal IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) where T : class { Ensure.Empty(_replicationDataMapperSelector, nameof(Consumes)); var methodInfo = handler @@ -61,4 +61,7 @@ public IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) _typeRegistry.Add(typeof(T), new Tuple(handler, methodInfo)); return new ConsumesTypedJsonTypedJsonOptionsContext(this); } + + public OptionsBuilder Consumes(IMessageHandler handler, Func opts) where T : class + => opts(Consumes(handler)); } diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index 590931f..4fa5b27 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -44,9 +44,10 @@ .MessageType("message_type") .MessageData("data") ) - .Consumes(consumer) - .JsonContext(SourceGenerationContext.Default) - .NamingPolicy(new AttributeNamingPolicy()) + .Consumes(consumer, opts => + opts + .WithJsonContext(SourceGenerationContext.Default) + .AndNamingPolicy(new AttributeNamingPolicy())) .ConsumesRawString(consumer) .ConsumesRawObject(consumer); } diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index 129844e..dee6dca 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -63,32 +63,38 @@ subscriptionOptions .ConnectionString(Settings.ConnectionString) .DataSource(provider.GetRequiredService()) - .WithReplicationOptions(new ReplicationSlotManagement.ReplicationSlotOptions($"{nameof(HandleImpl1)}_slot")) + .WithReplicationOptions( + new ReplicationSlotManagement.ReplicationSlotOptions($"{nameof(HandleImpl1)}_slot")) .WithPublicationOptions(new PublicationManagement.PublicationOptions($"{nameof(HandleImpl1)}_pub")) .WithErrorProcessor(provider.GetRequiredService()) - - .Consumes(provider.GetRequiredService>()) - .Consumes(provider.GetRequiredService>()) - .JsonContext(SourceGenerationContext.Default) - .NamingPolicy(provider.GetRequiredService()) - ) - .ResiliencyPipeline(provider.GetRequiredService>().GetPipeline("default")) + + .Consumes(provider.GetRequiredService>(), opts => + opts + .Consumes(provider.GetRequiredService>()) + .WithJsonContext(SourceGenerationContext.Default) + .AndNamingPolicy(provider.GetRequiredService())) + ) + .ResiliencyPipeline( + provider.GetRequiredService>().GetPipeline("default")) ) .AddBlumchen((provider, workerOptions) => workerOptions .Subscription(subscriptionOptions => subscriptionOptions.ConnectionString(Settings.ConnectionString) .DataSource(provider.GetRequiredService()) - .WithReplicationOptions(new ReplicationSlotManagement.ReplicationSlotOptions($"{nameof(HandleImpl2)}_slot")) + .WithReplicationOptions( + new ReplicationSlotManagement.ReplicationSlotOptions($"{nameof(HandleImpl2)}_slot")) .WithPublicationOptions(new PublicationManagement.PublicationOptions($"{nameof(HandleImpl2)}_pub")) .WithErrorProcessor(provider.GetRequiredService()) - - .Consumes(provider.GetRequiredService>()) - .JsonContext(SourceGenerationContext.Default) - .NamingPolicy(provider.GetRequiredService()) - ) - .ResiliencyPipeline(provider.GetRequiredService>().GetPipeline("default")) - ); + + .Consumes(provider.GetRequiredService>(), opts => + opts + .WithJsonContext(SourceGenerationContext.Default) + .AndNamingPolicy(provider.GetRequiredService()) + )) + .ResiliencyPipeline( + provider.GetRequiredService>().GetPipeline("default")) + ); await builder .Build() diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index b6ad753..31f6cc4 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -85,13 +85,14 @@ protected OptionsBuilder SetupFor( .DataSource(new NpgsqlDataSourceBuilder(connectionString).Build()) .ConnectionString(connectionString) - .Consumes(consumer) - .JsonContext(info) - .NamingPolicy(namingPolicy) + .Consumes(consumer, opts => opts + .WithJsonContext(info) + .AndNamingPolicy(namingPolicy)) .WithTable(o => o.Named(eventsTable)) .WithPublicationOptions( - new PublicationManagement.PublicationOptions(PublicationName: publicationName ?? Randomise("events_pub")) + new PublicationManagement.PublicationOptions( + PublicationName: publicationName ?? Randomise("events_pub")) ) .WithReplicationOptions( new ReplicationSlotManagement.ReplicationSlotOptions(slotName ?? Randomise("events_slot")) From cb460d1da9bc96906596fbccb17e373d5ef68013 Mon Sep 17 00:00:00 2001 From: giordanol Date: Thu, 1 Aug 2024 10:25:04 +0200 Subject: [PATCH 67/80] Added EnableSubscriptionAutoHeal - see https://github.com/event-driven-io/Blumchen/issues/27 --- docker-compose.yml | 2 + src/Blumchen/DependencyInjection/Worker.cs | 14 ++--- .../WorkerOptionsBuilder.cs | 42 ++++++++++++-- src/Blumchen/Subscriber/OptionsBuilder.cs | 2 +- .../Management/ReplicationSlotManagement.cs | 56 ++++++++++--------- src/Blumchen/Subscriptions/Subscription.cs | 2 +- src/Subscriber/Program.cs | 2 +- src/SubscriberWorker/Program.cs | 2 + 8 files changed, 80 insertions(+), 42 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fa2eee0..4d0e26f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,8 @@ services: - "wal_level=logical" - "-c" - "wal_compression=on" + - "-c" + - "max_slot_wal_keep_size=1" pgadmin: container_name: pgadmin_container image: dpage/pgadmin4 diff --git a/src/Blumchen/DependencyInjection/Worker.cs b/src/Blumchen/DependencyInjection/Worker.cs index bd5cba5..d4d52b5 100644 --- a/src/Blumchen/DependencyInjection/Worker.cs +++ b/src/Blumchen/DependencyInjection/Worker.cs @@ -32,16 +32,16 @@ static Action LoggerAction(LogLevel ll, bool enabled) protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await options.ResiliencePipeline.ExecuteAsync(async token => + await options.OuterPipeline.ExecuteAsync(async token => + await options.InnerPipeline.ExecuteAsync(async ct => { await using var subscription = new Subscription(); - await using var cursor = subscription.Subscribe(options.SubscriberOptions, ct: token) - .GetAsyncEnumerator(token); - Notify(logger, LogLevel.Information,"{WorkerName} started", WorkerName); - while (await cursor.MoveNextAsync().ConfigureAwait(false) && !token.IsCancellationRequested) + await using var cursor = subscription.Subscribe(options.SubscriberOptions, ct) + .GetAsyncEnumerator(ct); + Notify(logger, LogLevel.Information, "{WorkerName} started", WorkerName); + while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested) Notify(logger, LogLevel.Trace, "{cursor.Current} processed", cursor.Current); - - }, stoppingToken).ConfigureAwait(false); + }, token).ConfigureAwait(false), stoppingToken).ConfigureAwait(false); Notify(logger, LogLevel.Information, "{WorkerName} stopped", WorkerName); } diff --git a/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs b/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs index 750878a..b3ca9d5 100644 --- a/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs +++ b/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs @@ -1,37 +1,67 @@ using Blumchen.Subscriber; +using Blumchen.Subscriptions.Management; +using Npgsql; +using Npgsql.Replication; using Polly; namespace Blumchen.DependencyInjection; -public record WorkerOptions(ResiliencePipeline ResiliencePipeline, ISubscriberOptions SubscriberOptions); +public record WorkerOptions( + ISubscriberOptions SubscriberOptions, + ResiliencePipeline OuterPipeline, + ResiliencePipeline InnerPipeline); public interface IWorkerOptionsBuilder { IWorkerOptionsBuilder ResiliencyPipeline(ResiliencePipeline resiliencePipeline); IWorkerOptionsBuilder Subscription(Func? builder); WorkerOptions Build(); + IWorkerOptionsBuilder EnableSubscriptionAutoHeal(); } internal sealed class WorkerOptionsBuilder: IWorkerOptionsBuilder { - private ResiliencePipeline? _resiliencePipeline = default; + private ResiliencePipeline? _outerPipeline = default; + private Func? _innerPipelineFn = default; private Func? _builder; public IWorkerOptionsBuilder ResiliencyPipeline(ResiliencePipeline resiliencePipeline) { - _resiliencePipeline = resiliencePipeline; + _outerPipeline = resiliencePipeline; return this; }public IWorkerOptionsBuilder Subscription(Func? builder) { _builder = builder; - return this; + return this; } public WorkerOptions Build() { - ArgumentNullException.ThrowIfNull(_resiliencePipeline); + ArgumentNullException.ThrowIfNull(_outerPipeline); ArgumentNullException.ThrowIfNull(_builder); - return new(_resiliencePipeline, _builder(new OptionsBuilder()).Build()); + var subscriberOptions = _builder(new OptionsBuilder()).Build(); + return new(subscriberOptions, _outerPipeline, + _innerPipelineFn?.Invoke(subscriberOptions.ReplicationOptions.SlotName,subscriberOptions.ConnectionStringBuilder.ConnectionString) ?? + ResiliencePipeline.Empty + ); + } + + public IWorkerOptionsBuilder EnableSubscriptionAutoHeal() + { + _innerPipelineFn = (replicationSlotName, connectionString) => new ResiliencePipelineBuilder().AddRetry(new() + { + ShouldHandle = + new PredicateBuilder().Handle(exception => + exception.SqlState.Equals("55000", StringComparison.OrdinalIgnoreCase)), + MaxRetryAttempts = int.MaxValue, + OnRetry = async args => + { + await using var conn = new LogicalReplicationConnection(connectionString); + await conn.Open(args.Context.CancellationToken); + await conn.ReCreate(replicationSlotName, args.Context.CancellationToken).ConfigureAwait(false); + }, + }).Build(); + return this; } } diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index a83e141..a5d0a2d 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -149,7 +149,7 @@ internal ISubscriberOptions Build() if (_typeRegistry.Count > 0) { - Ensure.NotNull(_namingPolicy, $"{nameof(NamingPolicy)}"); + Ensure.NotNull(_namingPolicy, $"{nameof(NamingPolicy)}"); if (_jsonSerializerContext != null) { var typeResolver = new JsonTypeResolver(_jsonSerializerContext, _namingPolicy); diff --git a/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs b/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs index f95fb3a..180e2c6 100644 --- a/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs +++ b/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs @@ -28,37 +28,15 @@ CancellationToken ct { (Subscription.CreateStyle.Never,_) => new None(), (Subscription.CreateStyle.WhenNotExists,true) => new AlreadyExists(), - (Subscription.CreateStyle.WhenNotExists,false) => await Create(connection, slotName, ct).ConfigureAwait(false), - (Subscription.CreateStyle.AlwaysRecreate,true) => await ReCreate(connection, slotName, ct).ConfigureAwait(false), - (Subscription.CreateStyle.AlwaysRecreate, false) => await Create(connection, slotName, ct).ConfigureAwait(false), + (Subscription.CreateStyle.WhenNotExists,false) => await connection.Create(slotName, ct).ConfigureAwait(false), + (Subscription.CreateStyle.AlwaysRecreate,true) => await connection.ReCreate(slotName, ct).ConfigureAwait(false), + (Subscription.CreateStyle.AlwaysRecreate, false) => await connection.Create(slotName, ct).ConfigureAwait(false), _ => throw new ArgumentOutOfRangeException(nameof(options.CreateStyle)) }; - static async Task ReCreate( - LogicalReplicationConnection connection, - string slotName, - CancellationToken ct) - { - await connection.DropReplicationSlot(slotName, true, ct).ConfigureAwait(false); - return await Create(connection, slotName, ct).ConfigureAwait(false); - } - - static async Task Create( - LogicalReplicationConnection connection, - string slotName, - CancellationToken ct) - { - var result = await connection.CreatePgOutputReplicationSlot( - slotName, - slotSnapshotInitMode: LogicalSlotSnapshotInitMode.Export, - cancellationToken: ct - ).ConfigureAwait(false); - - return new Created(result.SnapshotName!, result.ConsistentPoint); - } } - + public record ReplicationSlotOptions( string SlotName = $"{TableDescriptorBuilder.MessageTable.DefaultName}_slot", Subscription.CreateStyle CreateStyle = Subscription.CreateStyle.WhenNotExists, @@ -74,3 +52,29 @@ public record AlreadyExists: CreateReplicationSlotResult; public record Created(string SnapshotName, NpgsqlLogSequenceNumber LogSequenceNumber): CreateReplicationSlotResult; } } + +public static class LogicalReplicationConnectionExtensions +{ + internal static async Task Create( + this LogicalReplicationConnection connection, + string slotName, + CancellationToken ct) + { + var result = await connection.CreatePgOutputReplicationSlot( + slotName, + slotSnapshotInitMode: LogicalSlotSnapshotInitMode.Export, + cancellationToken: ct + ).ConfigureAwait(false); + + return new Created(result.SnapshotName!, result.ConsistentPoint); + } + + public static async Task ReCreate( + this LogicalReplicationConnection connection, + string slotName, + CancellationToken ct) + { + await connection.DropReplicationSlot(slotName, true, ct).ConfigureAwait(false); + return await connection.Create(slotName, ct).ConfigureAwait(false); + } +} diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 4b5a43d..80c79cf 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -70,7 +70,7 @@ internal async IAsyncEnumerable Subscribe( await dataSource.SetupPublication(publicationSetupOptions, ct).ConfigureAwait(false); var result = await dataSource.SetupReplicationSlot(_connection, replicationSlotSetupOptions, ct) .ConfigureAwait(false); - IReplicationDataMapper replicationDataMapper = new ReplicationDataMapper(registry); + var replicationDataMapper = new ReplicationDataMapper(registry); PgOutputReplicationSlot slot; if (result is not Created created) diff --git a/src/Subscriber/Program.cs b/src/Subscriber/Program.cs index 4fa5b27..1cedf67 100644 --- a/src/Subscriber/Program.cs +++ b/src/Subscriber/Program.cs @@ -60,7 +60,7 @@ await using var cursor1 = cursor.ConfigureAwait(false); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested) if(logger.IsEnabled(LogLevel.Trace)) - logger.LogTrace($"{cursor.Current} processed"); + logger.LogTrace("{message} processed", cursor.Current); } catch (Exception e) { diff --git a/src/SubscriberWorker/Program.cs b/src/SubscriberWorker/Program.cs index dee6dca..93ec808 100644 --- a/src/SubscriberWorker/Program.cs +++ b/src/SubscriberWorker/Program.cs @@ -76,6 +76,7 @@ ) .ResiliencyPipeline( provider.GetRequiredService>().GetPipeline("default")) + .EnableSubscriptionAutoHeal() ) .AddBlumchen((provider, workerOptions) => workerOptions @@ -94,6 +95,7 @@ )) .ResiliencyPipeline( provider.GetRequiredService>().GetPipeline("default")) + .EnableSubscriptionAutoHeal() ); await builder From 80ee62c9a02906d04445f0080c489343cc140135 Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 2 Aug 2024 10:04:00 +0200 Subject: [PATCH 68/80] narrowed scope --- .../Subscriptions/Management/ReplicationSlotManagement.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs b/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs index 180e2c6..0ce5ee1 100644 --- a/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs +++ b/src/Blumchen/Subscriptions/Management/ReplicationSlotManagement.cs @@ -53,7 +53,7 @@ public record Created(string SnapshotName, NpgsqlLogSequenceNumber LogSequenceNu } } -public static class LogicalReplicationConnectionExtensions +internal static class LogicalReplicationConnectionExtensions { internal static async Task Create( this LogicalReplicationConnection connection, @@ -69,7 +69,7 @@ public static class LogicalReplicationConnectionExtensions return new Created(result.SnapshotName!, result.ConsistentPoint); } - public static async Task ReCreate( + internal static async Task ReCreate( this LogicalReplicationConnection connection, string slotName, CancellationToken ct) From 35bc4ff24554a6cd20625190018ca4cea39e299c Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 2 Aug 2024 10:24:44 +0200 Subject: [PATCH 69/80] enabling cli invocation --- src/Publisher/Program.cs | 74 +++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/src/Publisher/Program.cs b/src/Publisher/Program.cs index c0d8e7b..28cfaf5 100644 --- a/src/Publisher/Program.cs +++ b/src/Publisher/Program.cs @@ -27,41 +27,68 @@ .ConfigureAwait(false); //Or you might want to verify at a later stage +var loggerFactory = LoggerFactory.Create(builder => builder + .AddFilter("Microsoft", LogLevel.Warning) + .AddFilter("System", LogLevel.Warning) + .AddFilter("Npgsql", LogLevel.Information) + .AddFilter("Blumchen", LogLevel.Trace) + .AddFilter("Publisher", LogLevel.Debug) + .AddSimpleConsole()); +var logger = loggerFactory.CreateLogger(); await new NpgsqlDataSourceBuilder(Settings.ConnectionString) - .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())) + .UseLoggerFactory(loggerFactory) .Build() .EnsureTableExists(resolver.TableDescriptor, cts.Token).ConfigureAwait(false); -Parser.Default.ParseArguments("--help".Split()); -do -{ - async void GenerateFn(Options o) => await Generate(generator.Join(o.MessageTypes, pair => pair.Key, s => s, (pair, s) => pair).ToDictionary(), o.Count, resolver, cts); +//Uncomment to attach debugger +//System.Diagnostics.Debugger.Launch(); +async Task GenerateFn(Options o) => + await Generate(generator.Join(o.MessageTypes, pair => pair.Key, s => s, (pair, s) => pair).ToDictionary(), + o.Count, resolver, logger, cts); - Parser.Default.ParseArguments(Console.ReadLine()?.Split()) - .WithParsed(GenerateFn) +if (args.Length > 0) //cli +{ + Parser.Default.ParseArguments(args) + .WithParsed(options => Task.WaitAll([GenerateFn(options)])) .WithNotParsed(HandleParseError); +} +else +{ + Parser.Default.ParseArguments("--help".Split()); + do + { + Parser.Default.ParseArguments(Console.ReadLine()?.Split()) + .WithParsed(options => Task.WaitAll([GenerateFn(options)])) + .WithNotParsed(HandleParseError); + + } while (true); +} -} while (true); +return; static void HandleParseError(IEnumerable errs) => Console.WriteLine("Errors:" + string.Join(',', errs.Select(e => e.Tag))); -async Task Generate(Dictionary> dictionary, int result, PublisherOptions publisherOptions, +async Task Generate(Dictionary> dictionary, int count, PublisherOptions publisherOptions, + ILogger l, CancellationTokenSource cancellationTokenSource) { var generatorLength = dictionary.Count; - var messageCount = result / generatorLength; + var messageCount = count / generatorLength; var ct = cancellationTokenSource.Token; var connection = new NpgsqlConnection(Settings.ConnectionString); await using var connection1 = connection.ConfigureAwait(false); await connection.OpenAsync(ct).ConfigureAwait(false); //use a command for each message { - var tuple = Enumerable.Range(0, result).Select(i => + var tuple = Enumerable.Range(0, count).Select(i => dictionary.ElementAt(i % generatorLength)); - foreach (var s in dictionary.Keys.Select((key, i) => $"Publishing {(messageCount + (result % generatorLength > i ? 1 : 0))} {key}").ToList()) - await Console.Out.WriteLineAsync(s); + var messageByType = string.Join(", ", + dictionary.Keys.Select((key, i) => + $"Publishing {(messageCount + (count % generatorLength > i ? 1 : 0))} {key}")); + l.LogInformation(messageByType); + - foreach (var message in tuple.Select(pair => pair.Value())) + foreach (var message in tuple.Select(pair => pair.Value())/*.Chunk(10)*/)//Chunking enable batch insert { var transaction = await connection.BeginTransactionAsync(ct).ConfigureAwait(false); try @@ -89,27 +116,12 @@ async Task Generate(Dictionary> dictionary, int result, Pub catch (Exception e) { await transaction.RollbackAsync(ct).ConfigureAwait(false); - Console.WriteLine(e); + l.LogCritical(e, e.Message); throw; } } - await Console.Out.WriteLineAsync($"Published {result} messages!"); + l.LogInformation("Published {count} messages!", count); } - //use a batch command - //{ - // var transaction = await connection.BeginTransactionAsync(ct); - // try - // { - // var @events = Enumerable.Range(0, result) - // .Select(i1 => new UserCreated(Guid.NewGuid(), Guid.NewGuid().ToString())); - // await MessageAppender.AppendAsync(@events, resolver, connection, transaction, ct); - // } - // catch (Exception e) - // { - // Console.WriteLine(e); - // throw; - // } - //} } internal class Options(IEnumerable messageTypes, int count) From 0cf3e4881efabde44aaefa321d0e10708068ed7f Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 2 Aug 2024 10:25:18 +0200 Subject: [PATCH 70/80] set container name --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 4d0e26f..7cbbf38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ version: '3.8' # Specify Docker Compose version services: postgres: image: postgres:15.1-alpine + container_name: db ports: - "5432:5432" environment: From 7e16bbb30b2c2e1b3c3d5052486b9c0e66eef573 Mon Sep 17 00:00:00 2001 From: giordanol Date: Fri, 2 Aug 2024 10:33:32 +0200 Subject: [PATCH 71/80] added auto heal use case --- Blumchen.sln | 5 +++ scripts/autoheal.ps1 | 76 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 scripts/autoheal.ps1 diff --git a/Blumchen.sln b/Blumchen.sln index 6186db3..f2618e8 100644 --- a/Blumchen.sln +++ b/Blumchen.sln @@ -42,6 +42,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SubscriberWorker", "src\Sub EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "UnitTests", "src\UnitTests\UnitTests.csproj", "{B16305B4-8AC3-4435-AADB-D9E2ACAA1C13}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{13543764-C1E1-482E-887A-816ABA4CB71C}" + ProjectSection(SolutionItems) = preProject + scripts\autoheal.ps1 = scripts\autoheal.ps1 + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU diff --git a/scripts/autoheal.ps1 b/scripts/autoheal.ps1 new file mode 100644 index 0000000..bf78150 --- /dev/null +++ b/scripts/autoheal.ps1 @@ -0,0 +1,76 @@ +#usage: .\scripts\autoheal.ps1 -SolutionFile .\Blumchen.sln -ComposeFile .\docker-compose.yml +param( + [string]$SolutionFile, + [string]$ComposeFile +) + + +$env:DOCKER_CLI_HINTS=$false #disable docker hints +Write-Host "Setup infrastrucure" + +try { + + start powershell { + docker compose up; + Read-Host; + + } + + Write-Host "Waiting for container readiness..." + do + { + Start-Sleep -s 5 + $state=$(docker inspect db|ConvertFrom-Json).State + $status=$state.Status + $exitCode=$state.ExitCode + $restart=$state.Restarting + }Until(($status -eq "running") -and ($exitCode -eq 0) -and ($restart -eq $false)) + + Write-Host "...Done" + + Write-Host "Start subscriber" + start powershell { + dotnet run --project ./src/SubscriberWorker/SubscriberWorker.csproj + Read-Host; + } + + Write-Host "Publishing 10 messages to test the subscriptions are working properly: hit ENTER when done!" + + start powershell { + dotnet run --project ./src/Publisher/Publisher.csproj -- -c 10 -t \"UserCreated|UserDeleted|UserModified\"; + } + + Read-Host; + + Write-Host "Start massive insert to force wal segment creation..." + start powershell { + dotnet run --project ./src/Publisher/Publisher.csproj -- -c 800000 -t "UserSubscribed" + } + + Write-Host "Wait for subscribers to auto heal on error...reporting on row insert" + + Start-Sleep -s 15 + do + { + docker exec -it db psql -h localhost -U postgres -w -c "select count(*) from outbox;" + }Until(Read-Host "Enter to report on counting rows(another key to proceed when done)" "") + + Write-Host "Subscribers resiliency tested :-)" + Write-Host "Publishing 10 messages to test the subscriptions are still working properly: hit ENTER when done!" + + start powershell { + dotnet run --project ./src/Publisher/Publisher.csproj -- -c 10 -t \"UserCreated|UserDeleted|UserModified\" + } + Read-Host; + + Write-Host "We're done...: hit ENTER to shut down!" + + Read-Host; + +}catch { + Write-Host "An error occurred:" + Write-Host $_ +} +finally{ + docker compose -f $ComposeFile down --rmi local +} From 414538e9f8f04af6d7737e6abdb9753093fea240 Mon Sep 17 00:00:00 2001 From: giordanol Date: Sat, 17 Aug 2024 11:56:34 +0200 Subject: [PATCH 72/80] enforce single INamingPolicy instance per subscription --- .../Subscriber/OptionsBuilder.Consumes.cs | 4 ++- src/UnitTests/Contracts.cs | 7 ++++ src/UnitTests/subscriber_options_builder.cs | 34 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs index c37179b..71d0a11 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs @@ -14,6 +14,7 @@ public sealed partial class OptionsBuilder [UsedImplicitly] internal OptionsBuilder NamingPolicy(INamingPolicy namingPolicy) { + Ensure.Null(_namingPolicy, nameof(NamingPolicy)); _namingPolicy = namingPolicy; return this; } @@ -57,7 +58,8 @@ internal IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler .GetMethod(nameof(IMessageHandler.Handle), BindingFlags.Instance | BindingFlags.Public, [typeof(T)]) ?? throw new ConfigurationException($"Unable to find {nameof(IMessageHandler)} implementation on {handler.GetType().Name}"); - + if (_typeRegistry.ContainsKey(typeof(T))) + throw new ConfigurationException($"`{typeof(T).Name}` was already registered."); _typeRegistry.Add(typeof(T), new Tuple(handler, methodInfo)); return new ConsumesTypedJsonTypedJsonOptionsContext(this); } diff --git a/src/UnitTests/Contracts.cs b/src/UnitTests/Contracts.cs index cdeb01b..ca50595 100644 --- a/src/UnitTests/Contracts.cs +++ b/src/UnitTests/Contracts.cs @@ -9,6 +9,12 @@ public record UserCreatedContract( string Name ); + [MessageUrn("user-registered:v1")] + public record UserRegisteredContract( + Guid Id, + string Name + ); + [RawUrn("user-deleted:v1")] public class MessageObjects; @@ -20,5 +26,6 @@ public class InvalidMessage; [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(UserCreatedContract))] + [JsonSerializable(typeof(UserRegisteredContract))] internal partial class SourceGenerationContext: JsonSerializerContext; } diff --git a/src/UnitTests/subscriber_options_builder.cs b/src/UnitTests/subscriber_options_builder.cs index b7cb0dc..2a4240b 100644 --- a/src/UnitTests/subscriber_options_builder.cs +++ b/src/UnitTests/subscriber_options_builder.cs @@ -1,4 +1,5 @@ using Blumchen; +using Blumchen.Serialization; using Blumchen.Subscriber; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Replication; @@ -136,5 +137,38 @@ public void with_typed_raw_consumer_of_string_requires_RawUrn_decoration() Assert.IsType(exception); Assert.Equal("`RawUrnAttribute` missing on `InvalidMessage` message type", exception.Message); } + + [Fact] + public void does_not_allow_multiple_registration_of_the_same_typed_consumer() + { + var messageHandler = Substitute.For>(); + var exception = Record.Exception(() => _builder(ValidConnectionString) + .Consumes(messageHandler) + .WithJsonContext(SourceGenerationContext.Default) + .AndNamingPolicy(new AttributeNamingPolicy()) + .Consumes(messageHandler) + .WithJsonContext(SourceGenerationContext.Default) + .AndNamingPolicy(new AttributeNamingPolicy()) + .Build()); + Assert.IsType(exception); + Assert.Equal("`UserCreatedContract` was already registered.", exception.Message); + } + + [Fact] + public void with_typed_consumer_allows_only_one_naming_policy_instance() + { + var userCreatedMessageHandler = Substitute.For>(); + var userRegisteredMessageHandler = Substitute.For>(); + var exception = Record.Exception(() => _builder(ValidConnectionString) + .Consumes(userCreatedMessageHandler) + .WithJsonContext(SourceGenerationContext.Default) + .AndNamingPolicy(new AttributeNamingPolicy()) + .Consumes(userRegisteredMessageHandler) + .WithJsonContext(SourceGenerationContext.Default) + .AndNamingPolicy(new AttributeNamingPolicy()) + .Build()); + Assert.IsType(exception); + Assert.Equal("`NamingPolicy` method on OptionsBuilder called more then once", exception.Message); + } } } From cebdd98ad48c3844a5d59717683b683e552a083b Mon Sep 17 00:00:00 2001 From: giordanol Date: Sun, 18 Aug 2024 15:34:58 +0200 Subject: [PATCH 73/80] Added minimal ServiceWorker configuatrion with tests --- .../ServiceCollectionExtensions.cs | 19 +++- .../WorkerOptionsBuilder.cs | 8 +- src/Blumchen/Subscriber/IConsumes.cs | 15 +++ .../Subscriber/OptionsBuilder.Consumes.cs | 4 +- src/Blumchen/Subscriber/OptionsBuilder.cs | 15 +-- src/Blumchen/Subscriptions/IErrorProcessor.cs | 12 +++ src/Tests/DatabaseFixture.cs | 10 ++ src/Tests/ServiceCollectionExtensions.cs | 21 ++++ src/Tests/ServiceWorker..cs | 101 ++++++++++++++++++ src/Tests/SubscriberContext.cs | 3 + src/Tests/Tests.csproj | 2 + src/UnitTests/UnitTests.csproj | 1 + src/UnitTests/subscriber_options_builder.cs | 8 +- 13 files changed, 201 insertions(+), 18 deletions(-) create mode 100644 src/Blumchen/Subscriber/IConsumes.cs create mode 100644 src/Tests/ServiceCollectionExtensions.cs create mode 100644 src/Tests/ServiceWorker..cs diff --git a/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs index b1a2081..91e08ad 100644 --- a/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs @@ -1,6 +1,9 @@ +using Blumchen.Subscriber; using Blumchen.Subscriptions.Replication; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Npgsql; #pragma warning disable IL2091 @@ -14,10 +17,24 @@ public static IServiceCollection AddBlumchen( Func workerOptions) where T : class, IMessageHandler => service - .AddKeyedSingleton(typeof(T), (provider, _) => workerOptions(provider, new WorkerOptionsBuilder()).Build()) .AddHostedService(provider => new Worker(workerOptions(provider, new WorkerOptionsBuilder()).Build(), provider.GetRequiredService>>())); + public static IServiceCollection AddBlumchen( + this IServiceCollection service, + string connectionString, + Func consumerFn) where T : class, IMessageHandler { + return service + .AddHostedService(provider => + new Worker(MinimalWorkerOptions(provider, new WorkerOptionsBuilder()).Build(), + provider.GetService>>() ?? new NullLogger>())); + + IWorkerOptionsBuilder MinimalWorkerOptions(IServiceProvider provider, IWorkerOptionsBuilder builder) + => builder.Subscription(optionsBuilder => consumerFn(provider, optionsBuilder) + .ConnectionString(connectionString) + .DataSource(NpgsqlDataSource.Create(connectionString))); + + } } diff --git a/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs b/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs index b3ca9d5..45bb272 100644 --- a/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs +++ b/src/Blumchen/DependencyInjection/WorkerOptionsBuilder.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.CodeAnalysis; using Blumchen.Subscriber; using Blumchen.Subscriptions.Management; using Npgsql; @@ -21,8 +22,8 @@ public interface IWorkerOptionsBuilder internal sealed class WorkerOptionsBuilder: IWorkerOptionsBuilder { - private ResiliencePipeline? _outerPipeline = default; - private Func? _innerPipelineFn = default; + private ResiliencePipeline? _outerPipeline; + private Func? _innerPipelineFn; private Func? _builder; public IWorkerOptionsBuilder ResiliencyPipeline(ResiliencePipeline resiliencePipeline) @@ -37,10 +38,9 @@ public IWorkerOptionsBuilder ResiliencyPipeline(ResiliencePipeline resiliencePip public WorkerOptions Build() { - ArgumentNullException.ThrowIfNull(_outerPipeline); ArgumentNullException.ThrowIfNull(_builder); var subscriberOptions = _builder(new OptionsBuilder()).Build(); - return new(subscriberOptions, _outerPipeline, + return new(subscriberOptions, _outerPipeline ?? ResiliencePipeline.Empty, _innerPipelineFn?.Invoke(subscriberOptions.ReplicationOptions.SlotName,subscriberOptions.ConnectionStringBuilder.ConnectionString) ?? ResiliencePipeline.Empty ); diff --git a/src/Blumchen/Subscriber/IConsumes.cs b/src/Blumchen/Subscriber/IConsumes.cs new file mode 100644 index 0000000..41b8581 --- /dev/null +++ b/src/Blumchen/Subscriber/IConsumes.cs @@ -0,0 +1,15 @@ +using Blumchen.Subscriptions.Replication; +using static Blumchen.Subscriber.OptionsBuilder; + +namespace Blumchen.Subscriber; + +public interface IConsumes +{ + OptionsBuilder ConsumesRawStrings(IMessageHandler handler); + OptionsBuilder ConsumesRawObjects(IMessageHandler handler); + OptionsBuilder ConsumesRawString(IMessageHandler handler) where T : class; + OptionsBuilder ConsumesRawObject(IMessageHandler handler) where T : class; + + OptionsBuilder Consumes(IMessageHandler handler, Func opts) + where T : class; +} diff --git a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs index 71d0a11..f5035c5 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs @@ -46,7 +46,7 @@ public INamingOptionsContext WithJsonContext(JsonSerializerContext jsonSerialize public IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler) where T : class { - return builder.Consumes(handler); + return builder.Consumes(handler); } } @@ -65,5 +65,5 @@ internal IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler } public OptionsBuilder Consumes(IMessageHandler handler, Func opts) where T : class - => opts(Consumes(handler)); + => opts(Consumes(handler)); } diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index a5d0a2d..7203b1d 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -10,6 +10,7 @@ namespace Blumchen.Subscriber; public sealed partial class OptionsBuilder + : IConsumes { internal const string WildCard = "*"; @@ -75,11 +76,11 @@ public OptionsBuilder WithReplicationOptions( } [UsedImplicitly] - public OptionsBuilder ConsumesRawObject(IMessageHandler handler) where T : class + public OptionsBuilder ConsumesRawObject(IMessageHandler handler) where T : class => ConsumesRaw(handler, ObjectReplicationDataMapper.Instance); [UsedImplicitly] - public OptionsBuilder ConsumesRawString(IMessageHandler handler) where T : class + public OptionsBuilder ConsumesRawString(IMessageHandler handler) where T : class => ConsumesRaw(handler, StringReplicationDataMapper.Instance); [UsedImplicitly] @@ -96,16 +97,16 @@ public OptionsBuilder ConsumesRawStrings(IMessageHandler handler) new Tuple(StringReplicationDataMapper.Instance, handler, methodInfo)); return this; } - + [UsedImplicitly] - public OptionsBuilder ConsumesRawObjects(IMessageHandler handler) + public OptionsBuilder ConsumesRawObjects(IMessageHandler handler) { Ensure.Empty(_replicationDataMapperSelector, _typeRegistry, nameof(ConsumesRawObjects)); var methodInfo = handler .GetType() - .GetMethod(nameof(IMessageHandler.Handle), BindingFlags.Instance | BindingFlags.Public, [typeof(string)]) - ?? throw new ConfigurationException($"Unable to find {nameof(IMessageHandler)} implementation on {handler.GetType().Name}"); + .GetMethod(nameof(IMessageHandler.Handle), BindingFlags.Instance | BindingFlags.Public, [typeof(object)]) + ?? throw new ConfigurationException($"Unable to find {nameof(IMessageHandler)} implementation on {handler.GetType().Name}"); _replicationDataMapperSelector.Add(WildCard, @@ -113,7 +114,7 @@ public OptionsBuilder ConsumesRawObjects(IMessageHandler handler) return this; } - private OptionsBuilder ConsumesRaw(IMessageHandler handler, + private OptionsBuilder ConsumesRaw(IMessageHandler handler, IReplicationJsonBMapper dataMapper) where T : class where TU : class { var urns = typeof(T) diff --git a/src/Blumchen/Subscriptions/IErrorProcessor.cs b/src/Blumchen/Subscriptions/IErrorProcessor.cs index 527c579..889d97d 100644 --- a/src/Blumchen/Subscriptions/IErrorProcessor.cs +++ b/src/Blumchen/Subscriptions/IErrorProcessor.cs @@ -1,3 +1,5 @@ +using Microsoft.Extensions.Logging; + namespace Blumchen.Subscriptions; public interface IErrorProcessor @@ -9,3 +11,13 @@ public record ConsoleOutErrorProcessor: IErrorProcessor { public Func Process => (exception, id) => Console.Out.WriteLineAsync($"record id:{id} resulted in error:{exception.Message}"); } + +public record LoggingErrorProcessor(ILogger Logger): IErrorProcessor +{ + public Func Process => (exception, id) + => + { + Logger.LogError("record id:{id} resulted in error:{exception.Message}", id, exception.Message); + return Task.CompletedTask; + }; +} diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index 31f6cc4..344c6c6 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -8,6 +8,7 @@ using Blumchen.Subscriptions; using Blumchen.Subscriptions.Management; using Blumchen.Subscriptions.Replication; +using Microsoft.Extensions.Logging; using Npgsql; using Testcontainers.PostgreSql; using Xunit.Abstractions; @@ -34,6 +35,15 @@ public async Task Handle(T value) } } + protected class TestHandler(ILogger> logger): IMessageHandler where T : class + { + public Task Handle(T value) + { + logger.LogTrace(value.ToString()); + return Task.CompletedTask; + } + } + protected readonly PostgreSqlContainer Container = new PostgreSqlBuilder() .WithCommand("-c", "wal_level=logical") .Build(); diff --git a/src/Tests/ServiceCollectionExtensions.cs b/src/Tests/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..b5e738a --- /dev/null +++ b/src/Tests/ServiceCollectionExtensions.cs @@ -0,0 +1,21 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace Tests; + +internal static class ServiceCollectionExtensions +{ + internal static IServiceCollection AddXunitLogging(this IServiceCollection services, ITestOutputHelper output) => + services + .AddLogging(loggingBuilder => + { + loggingBuilder + .AddFilter("Microsoft", LogLevel.Warning) + .AddFilter("System", LogLevel.Warning) + .AddFilter("Npgsql", LogLevel.Information) + .AddFilter("Blumchen", LogLevel.Trace) + .AddFilter("SubscriberWorker", LogLevel.Debug) + .AddXunit(output); + }); +} diff --git a/src/Tests/ServiceWorker..cs b/src/Tests/ServiceWorker..cs new file mode 100644 index 0000000..e0e644b --- /dev/null +++ b/src/Tests/ServiceWorker..cs @@ -0,0 +1,101 @@ +using Blumchen.DependencyInjection; +using Blumchen.Publisher; +using Blumchen.Serialization; +using Blumchen.Subscriber; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Xunit.Abstractions; +using SubscriberOptionsBuilder = Blumchen.Subscriber.OptionsBuilder; +using PublisherOptionsBuilder = Blumchen.Publisher.OptionsBuilder; + +namespace Tests +{ + public class ServiceWorker(ITestOutputHelper testOutputHelper): DatabaseFixture(testOutputHelper) + { + [Fact] + public async Task ConsumesRawStrings() => await Consumes( + (services, opts) => opts.ConsumesRawStrings(services.GetRequiredService>()) + ); + + [Fact] + public async Task ConsumesRawObjects() => + await Consumes( + (services,opts) => opts.ConsumesRawObjects(services.GetRequiredService>()) + ); + + [Fact] + public async Task ConsumesRawString() => await Consumes( + (services, opts) => opts.ConsumesRawString(services.GetRequiredService>()) + ); + + [Fact] + public async Task ConsumesRawObject() => await Consumes( + (services, opts) => + { + var handler = services.GetRequiredService>(); + return opts.ConsumesRawObject(handler); + }); + + + + [Fact] + public async Task ConsumesJson_without_shared_kernel() => await Consumes( + (services, builder) => builder + .Consumes(services.GetRequiredService>(), + opts => opts + .WithJsonContext(SubscriberContext.Default) + .AndNamingPolicy(new AttributeNamingPolicy()) + ) + ); + + [Fact] + public async Task ConsumesJson_with_shared_kernel() + { + var namingPolicy = new FQNNamingPolicy(); + await Consumes( + (services, builder) => builder + .Consumes(services.GetRequiredService>(), + opts => opts + .WithJsonContext(PublisherContext.Default) + .AndNamingPolicy(namingPolicy) + ), namingPolicy + ); + } + + private async Task Consumes( + Func consumesFn, + INamingPolicy? namingPolicy = default + ) where T : class + { + var ct = new CancellationTokenSource(TimeSpan.FromSeconds(3)); + + var options = await new PublisherOptionsBuilder() + .JsonContext(PublisherContext.Default) + .NamingPolicy(namingPolicy ?? new AttributeNamingPolicy()) + .Build() + .EnsureTable(Container.GetConnectionString(), ct.Token); + + await MessageAppender.AppendAsync( + new PublisherUserCreated(Guid.NewGuid(), nameof(PublisherUserCreated)), + options, + Container.GetConnectionString(), + ct.Token + ); + + var builder = Host.CreateApplicationBuilder(); + builder.Services + .AddXunitLogging(Output) + .AddSingleton>() + .AddBlumchen>( + Container.GetConnectionString(), + consumesFn + ); + + await builder + .Build() + .RunAsync(ct.Token); + + } + } +} diff --git a/src/Tests/SubscriberContext.cs b/src/Tests/SubscriberContext.cs index 892d0a8..36d0a00 100644 --- a/src/Tests/SubscriberContext.cs +++ b/src/Tests/SubscriberContext.cs @@ -18,3 +18,6 @@ string Name [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(SubscriberUserCreated))] internal partial class SubscriberContext: JsonSerializerContext; + +[RawUrn("user-created:v1")] +internal class DecoratedContract; diff --git a/src/Tests/Tests.csproj b/src/Tests/Tests.csproj index 342eda0..59e3a57 100644 --- a/src/Tests/Tests.csproj +++ b/src/Tests/Tests.csproj @@ -7,7 +7,9 @@ + + diff --git a/src/UnitTests/UnitTests.csproj b/src/UnitTests/UnitTests.csproj index 17ace07..055e70f 100644 --- a/src/UnitTests/UnitTests.csproj +++ b/src/UnitTests/UnitTests.csproj @@ -8,6 +8,7 @@ + diff --git a/src/UnitTests/subscriber_options_builder.cs b/src/UnitTests/subscriber_options_builder.cs index 2a4240b..15c6e6a 100644 --- a/src/UnitTests/subscriber_options_builder.cs +++ b/src/UnitTests/subscriber_options_builder.cs @@ -123,8 +123,8 @@ public void ConsumesRawStrings_cannot_be_mixed_with_other_consuming_strategies() [Fact] public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() { - var messageHandler = Substitute.For>(); - var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawObject(messageHandler).Build()); + var messageHandler = Substitute.For>(); + var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawObject(messageHandler).Build()); Assert.IsType(exception); Assert.Equal("`RawUrnAttribute` missing on `InvalidMessage` message type", exception.Message); } @@ -132,8 +132,8 @@ public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() [Fact] public void with_typed_raw_consumer_of_string_requires_RawUrn_decoration() { - var messageHandler = Substitute.For>(); - var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawString(messageHandler).Build()); + var messageHandler = Substitute.For>(); + var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawString(messageHandler).Build()); Assert.IsType(exception); Assert.Equal("`RawUrnAttribute` missing on `InvalidMessage` message type", exception.Message); } From 6cbcf1e3144a9bd456a8050579c311e8347f4de7 Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 19 Aug 2024 09:02:36 +0200 Subject: [PATCH 74/80] Added string routing capability and renamed to ([Message|Raw])RoutedBy... to alingn with semantic. --- .../ServiceCollectionExtensions.cs | 3 +- src/Blumchen/Ensure.cs | 2 +- ...UrnAttribute.cs => RoutingByAttributes.cs} | 56 ++++++++++++------- src/Blumchen/Subscriber/OptionsBuilder.cs | 10 ++-- src/Publisher/Contracts.cs | 8 +-- src/Subscriber/Contracts.cs | 6 +- src/SubscriberWorker/Contracts.cs | 6 +- src/Tests/DatabaseFixture.cs | 4 +- src/Tests/PublisherContext.cs | 4 +- src/Tests/ServiceCollectionExtensions.cs | 18 ++---- src/Tests/ServiceWorker..cs | 15 ++++- src/Tests/SubscriberContext.cs | 7 ++- src/UnitTests/Contracts.cs | 8 +-- src/UnitTests/subscriber_options_builder.cs | 4 +- 14 files changed, 88 insertions(+), 63 deletions(-) rename src/Blumchen/Serialization/{MessageUrnAttribute.cs => RoutingByAttributes.cs} (54%) diff --git a/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs index 91e08ad..daf6ac3 100644 --- a/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/Blumchen/DependencyInjection/ServiceCollectionExtensions.cs @@ -33,7 +33,8 @@ public static IServiceCollection AddBlumchen( IWorkerOptionsBuilder MinimalWorkerOptions(IServiceProvider provider, IWorkerOptionsBuilder builder) => builder.Subscription(optionsBuilder => consumerFn(provider, optionsBuilder) .ConnectionString(connectionString) - .DataSource(NpgsqlDataSource.Create(connectionString))); + .DataSource(new NpgsqlDataSourceBuilder(connectionString) + .UseLoggerFactory(provider.GetService()).Build())); } diff --git a/src/Blumchen/Ensure.cs b/src/Blumchen/Ensure.cs index 4cfeb05..75d8562 100644 --- a/src/Blumchen/Ensure.cs +++ b/src/Blumchen/Ensure.cs @@ -25,7 +25,7 @@ public bool IsValid(T value, params string[] parameters) } } -internal class RawUrnTrait(): Validable(v => v is ICollection { Count: > 0 }, $"`{nameof(RawUrnAttribute)}` missing on `{typeof(TR).Name}` message type"); +internal class RawUrnTrait(): Validable(v => v is ICollection { Count: > 0 }, $"`{nameof(RawRoutedByUrnAttribute)}` missing on `{typeof(TR).Name}` message type"); internal class NullTrait(): Validable(v => v is null, $"`{{0}}` method on {nameof(OptionsBuilder)} called more then once"); internal class NotNullTrait(): Validable(v => v is not null, $"`{{0}}` method not called on {nameof(OptionsBuilder)}"); internal class NotEmptyTrait(): Validable(v => v is ICollection { Count: > 0 }, $"No `{{0}}` method called on {nameof(OptionsBuilder)}"); diff --git a/src/Blumchen/Serialization/MessageUrnAttribute.cs b/src/Blumchen/Serialization/RoutingByAttributes.cs similarity index 54% rename from src/Blumchen/Serialization/MessageUrnAttribute.cs rename to src/Blumchen/Serialization/RoutingByAttributes.cs index dfc2db7..36f69ef 100644 --- a/src/Blumchen/Serialization/MessageUrnAttribute.cs +++ b/src/Blumchen/Serialization/RoutingByAttributes.cs @@ -2,25 +2,27 @@ namespace Blumchen.Serialization; +public interface IRouted +{ + string Route { get; } +} + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] -public class MessageUrnAttribute: - Attribute +public class MessageRoutedByUrnAttribute(string route): + Attribute, IRouted { - /// - /// - /// The urn value to use for this message type. - public MessageUrnAttribute(string urn) + public string Route { get; } = Format(route); + + private static string Format(string urn) { ArgumentException.ThrowIfNullOrEmpty(urn, nameof(urn)); if (urn.StartsWith(MessageUrn.Prefix)) throw new ArgumentException($"Value should not contain the default prefix '{MessageUrn.Prefix}'.", nameof(urn)); - Urn = FormatUrn(urn); + return FormatUrn(urn).AbsoluteUri; } - public Uri Urn { get; } - private static Uri FormatUrn(string urn) { var fullValue = MessageUrn.Prefix + urn; @@ -32,30 +34,44 @@ private static Uri FormatUrn(string urn) } } -[AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = false)] -public class RawUrnAttribute(string urn): MessageUrnAttribute(urn); - -public static class MessageUrn +internal static class MessageUrn { public const string Prefix = "urn:message:"; private static readonly ConcurrentDictionary Cache = new(); - - + public static string ForTypeString(Type type) => - Cache.GetOrAdd(type,t => + Cache.GetOrAdd(type, t => { - var attribute = Attribute.GetCustomAttribute(t, typeof(MessageUrnAttribute)) as MessageUrnAttribute ?? + var attribute = Attribute.GetCustomAttribute(t, typeof(MessageRoutedByUrnAttribute)) as MessageRoutedByUrnAttribute ?? throw new NotSupportedException($"Attribute not defined fot type '{type}'"); - return new Cached(attribute.Urn, attribute.Urn.ToString()); + return new Cached(attribute.Route); }).UrnString; private interface ICached { - Uri Urn { get; } string UrnString { get; } } - private record Cached(Uri Urn, string UrnString): ICached; + private record Cached(string UrnString): ICached; } + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = false)] +public class RawRoutedByUrnAttribute(string route): MessageRoutedByUrnAttribute(route); + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = false)] +public class RawRoutedByStringAttribute(string name): Attribute, IRouted +{ + private static string Format(string name) + { + if(string.IsNullOrWhiteSpace(name)) + throw new FormatException($"Invalid {nameof(name)}: {name}."); + + return name; + } + public string Route { get; } = Format(name); +} + + + diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index 7203b1d..01c7a13 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -118,10 +118,12 @@ private OptionsBuilder ConsumesRaw(IMessageHandler handler, IReplicationJsonBMapper dataMapper) where T : class where TU : class { var urns = typeof(T) - .GetCustomAttributes(typeof(RawUrnAttribute), false) - .OfType() - .Select(attribute => attribute.Urn).ToList(); - Ensure.RawUrn,T>(urns, nameof(NamingPolicy)); + .GetCustomAttributes(typeof(RawRoutedByUrnAttribute), false) + .Union(typeof(T) + .GetCustomAttributes(typeof(RawRoutedByStringAttribute), false)) + .OfType() + .Select(attribute => attribute.Route).ToList(); + Ensure.RawUrn,T>(urns, nameof(NamingPolicy)); var methodInfo = handler .GetType() diff --git a/src/Publisher/Contracts.cs b/src/Publisher/Contracts.cs index 8a10b46..2ef3048 100644 --- a/src/Publisher/Contracts.cs +++ b/src/Publisher/Contracts.cs @@ -4,25 +4,25 @@ namespace Publisher; public interface IContract{} -[MessageUrn("user-created:v1")] +[MessageRoutedByUrn("user-created:v1")] internal record UserCreated( Guid Id, string Name = "Created" ):IContract; -[MessageUrn("user-deleted:v1")] +[MessageRoutedByUrn("user-deleted:v1")] internal record UserDeleted( Guid Id, string Name = "Deleted" ): IContract; -[MessageUrn("user-modified:v1")] +[MessageRoutedByUrn("user-modified:v1")] internal record UserModified( Guid Id, string Name = "Modified" ): IContract; -[MessageUrn("user-subscribed:v1")] +[MessageRoutedByUrn("user-subscribed:v1")] internal record UserSubscribed( Guid Id, string Name = "Subscribed" diff --git a/src/Subscriber/Contracts.cs b/src/Subscriber/Contracts.cs index 19e99ba..e3c03ae 100644 --- a/src/Subscriber/Contracts.cs +++ b/src/Subscriber/Contracts.cs @@ -3,17 +3,17 @@ namespace Subscriber { - [MessageUrn("user-created:v1")] + [MessageRoutedByUrn("user-created:v1")] public record UserCreatedContract( Guid Id, string Name ); - [RawUrn("user-deleted:v1")] + [RawRoutedByUrn("user-deleted:v1")] public class MessageObjects; - [RawUrn("user-modified:v1")] + [RawRoutedByUrn("user-modified:v1")] internal class MessageString; [JsonSourceGenerationOptions(WriteIndented = true)] diff --git a/src/SubscriberWorker/Contracts.cs b/src/SubscriberWorker/Contracts.cs index a3c501a..fc4719a 100644 --- a/src/SubscriberWorker/Contracts.cs +++ b/src/SubscriberWorker/Contracts.cs @@ -3,19 +3,19 @@ namespace SubscriberWorker { - [MessageUrn("user-created:v1")] + [MessageRoutedByUrn("user-created:v1")] public record UserCreatedContract( Guid Id, string Name ); - [MessageUrn("user-deleted:v1")] + [MessageRoutedByUrn("user-deleted:v1")] public record UserDeletedContract( Guid Id, string Name ); - [MessageUrn("user-modified:v1")] //subscription ignored + [MessageRoutedByUrn("user-modified:v1")] //subscription ignored public record UserModifiedContract( Guid Id, string Name = "Modified" diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index 344c6c6..eb27253 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -18,7 +18,7 @@ namespace Tests; public abstract class DatabaseFixture(ITestOutputHelper output): IAsyncLifetime { protected ITestOutputHelper Output { get; } = output; - protected readonly Func TimeoutTokenSource = () => new(Debugger.IsAttached ? TimeSpan.FromHours(1) : TimeSpan.FromSeconds(2)); + protected readonly Func TimeoutTokenSource = () => new(Debugger.IsAttached ? TimeSpan.FromHours(1) : TimeSpan.FromSeconds(3)); protected class TestMessageHandler(Action log, JsonTypeInfo info): IMessageHandler where T : class { public async Task Handle(T value) @@ -39,7 +39,7 @@ protected class TestHandler(ILogger> logger): IMessageHandler< { public Task Handle(T value) { - logger.LogTrace(value.ToString()); + logger.LogTrace($"Message consumed:{value}"); return Task.CompletedTask; } } diff --git a/src/Tests/PublisherContext.cs b/src/Tests/PublisherContext.cs index a35dd25..b8bbc3f 100644 --- a/src/Tests/PublisherContext.cs +++ b/src/Tests/PublisherContext.cs @@ -3,13 +3,13 @@ namespace Tests; -[MessageUrn("user-created:v1")] +[MessageRoutedByUrn("user-created:v1")] internal record PublisherUserCreated( Guid Id, string Name ); -[MessageUrn("user-deleted:v1")] +[MessageRoutedByUrn("user-deleted:v1")] internal record PublisherUserDeleted( Guid Id, string Name diff --git a/src/Tests/ServiceCollectionExtensions.cs b/src/Tests/ServiceCollectionExtensions.cs index b5e738a..f0b1d23 100644 --- a/src/Tests/ServiceCollectionExtensions.cs +++ b/src/Tests/ServiceCollectionExtensions.cs @@ -6,16 +6,10 @@ namespace Tests; internal static class ServiceCollectionExtensions { - internal static IServiceCollection AddXunitLogging(this IServiceCollection services, ITestOutputHelper output) => - services - .AddLogging(loggingBuilder => - { - loggingBuilder - .AddFilter("Microsoft", LogLevel.Warning) - .AddFilter("System", LogLevel.Warning) - .AddFilter("Npgsql", LogLevel.Information) - .AddFilter("Blumchen", LogLevel.Trace) - .AddFilter("SubscriberWorker", LogLevel.Debug) - .AddXunit(output); - }); + internal static IServiceCollection AddXunitLogging(this IServiceCollection services, ITestOutputHelper output) + => services.AddLogging(loggingBuilder => + loggingBuilder + .AddFilter("Tests", LogLevel.Trace) + .AddXunit(output) + ); } diff --git a/src/Tests/ServiceWorker..cs b/src/Tests/ServiceWorker..cs index e0e644b..675edd6 100644 --- a/src/Tests/ServiceWorker..cs +++ b/src/Tests/ServiceWorker..cs @@ -7,6 +7,7 @@ using Xunit.Abstractions; using SubscriberOptionsBuilder = Blumchen.Subscriber.OptionsBuilder; using PublisherOptionsBuilder = Blumchen.Publisher.OptionsBuilder; +using Microsoft.Extensions.Logging; namespace Tests { @@ -35,9 +36,7 @@ public async Task ConsumesRawObject() => await Consumes( var handler = services.GetRequiredService>(); return opts.ConsumesRawObject(handler); }); - - [Fact] public async Task ConsumesJson_without_shared_kernel() => await Consumes( (services, builder) => builder @@ -62,13 +61,23 @@ await Consumes( ); } + [Fact] + public async Task ConsumesRawString_from_FQNNaming() + { + await Consumes( + (services, builder) => builder + .ConsumesRawString(services.GetRequiredService>() + ), new FQNNamingPolicy() + ); + } + private async Task Consumes( Func consumesFn, INamingPolicy? namingPolicy = default ) where T : class { - var ct = new CancellationTokenSource(TimeSpan.FromSeconds(3)); + var ct = TimeoutTokenSource(); var options = await new PublisherOptionsBuilder() .JsonContext(PublisherContext.Default) diff --git a/src/Tests/SubscriberContext.cs b/src/Tests/SubscriberContext.cs index 36d0a00..52012e1 100644 --- a/src/Tests/SubscriberContext.cs +++ b/src/Tests/SubscriberContext.cs @@ -4,7 +4,7 @@ namespace Tests; -[MessageUrn("user-created:v1")] +[MessageRoutedByUrn("user-created:v1")] internal record SubscriberUserCreated( Guid Id, string Name @@ -19,5 +19,8 @@ string Name [JsonSerializable(typeof(SubscriberUserCreated))] internal partial class SubscriberContext: JsonSerializerContext; -[RawUrn("user-created:v1")] +[RawRoutedByUrn("user-created:v1")] +[RawRoutedByString("Tests.PublisherUserCreated")] internal class DecoratedContract; + + diff --git a/src/UnitTests/Contracts.cs b/src/UnitTests/Contracts.cs index ca50595..e295820 100644 --- a/src/UnitTests/Contracts.cs +++ b/src/UnitTests/Contracts.cs @@ -3,23 +3,23 @@ namespace UnitTests { - [MessageUrn("user-created:v1")] + [MessageRoutedByUrn("user-created:v1")] public record UserCreatedContract( Guid Id, string Name ); - [MessageUrn("user-registered:v1")] + [MessageRoutedByUrn("user-registered:v1")] public record UserRegisteredContract( Guid Id, string Name ); - [RawUrn("user-deleted:v1")] + [RawRoutedByUrn("user-deleted:v1")] public class MessageObjects; - [RawUrn("user-modified:v1")] + [RawRoutedByUrn("user-modified:v1")] internal class MessageString; public class InvalidMessage; diff --git a/src/UnitTests/subscriber_options_builder.cs b/src/UnitTests/subscriber_options_builder.cs index 15c6e6a..7bd0556 100644 --- a/src/UnitTests/subscriber_options_builder.cs +++ b/src/UnitTests/subscriber_options_builder.cs @@ -126,7 +126,7 @@ public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() var messageHandler = Substitute.For>(); var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawObject(messageHandler).Build()); Assert.IsType(exception); - Assert.Equal("`RawUrnAttribute` missing on `InvalidMessage` message type", exception.Message); + Assert.Equal($"`{nameof(RawRoutedByUrnAttribute)}` missing on `InvalidMessage` message type", exception.Message); } [Fact] @@ -135,7 +135,7 @@ public void with_typed_raw_consumer_of_string_requires_RawUrn_decoration() var messageHandler = Substitute.For>(); var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawString(messageHandler).Build()); Assert.IsType(exception); - Assert.Equal("`RawUrnAttribute` missing on `InvalidMessage` message type", exception.Message); + Assert.Equal($"`{nameof(RawRoutedByUrnAttribute)}` missing on `InvalidMessage` message type", exception.Message); } [Fact] From be963fb703c7003ff1ceaaf9a1d28c08f20f8962 Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 19 Aug 2024 10:50:43 +0200 Subject: [PATCH 75/80] move to source generated logger --- .../DependencyInjection/LoggerExtensions.cs | 18 +++++++++++++ src/Blumchen/DependencyInjection/Worker.cs | 26 +++---------------- 2 files changed, 21 insertions(+), 23 deletions(-) create mode 100644 src/Blumchen/DependencyInjection/LoggerExtensions.cs diff --git a/src/Blumchen/DependencyInjection/LoggerExtensions.cs b/src/Blumchen/DependencyInjection/LoggerExtensions.cs new file mode 100644 index 0000000..721690f --- /dev/null +++ b/src/Blumchen/DependencyInjection/LoggerExtensions.cs @@ -0,0 +1,18 @@ +using Blumchen.Subscriptions.Replication; +using Microsoft.Extensions.Logging; + +namespace Blumchen.DependencyInjection; + +internal static partial class LoggerExtensions + +{ + [LoggerMessage(Message = "{workerName} started", Level = LogLevel.Information)] + public static partial void ServiceStarted(this ILogger logger, string workerName); + + [LoggerMessage(Message = "{workerName} sopped", Level = LogLevel.Information)] + public static partial void ServiceStopped(this ILogger logger, string workerName); + + [LoggerMessage(Message = "{message} processed", Level = LogLevel.Trace)] + public static partial void MessageProcessed(this ILogger logger, IEnvelope message); + +} diff --git a/src/Blumchen/DependencyInjection/Worker.cs b/src/Blumchen/DependencyInjection/Worker.cs index d4d52b5..3732620 100644 --- a/src/Blumchen/DependencyInjection/Worker.cs +++ b/src/Blumchen/DependencyInjection/Worker.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Replication; using Microsoft.Extensions.Hosting; @@ -11,25 +10,6 @@ public class Worker( ILogger> logger): BackgroundService where T : class, IMessageHandler { private string WorkerName { get; } = $"{nameof(Worker)}<{typeof(T).Name}>"; - private static readonly ConcurrentDictionary> LoggingActions = new(StringComparer.OrdinalIgnoreCase); - private static void Notify(ILogger logger, LogLevel level, string template, params object[] parameters) - { - LoggingActions.GetOrAdd(template,_ => LoggerAction(level, logger.IsEnabled(level)))(logger, template, parameters); - return; - - static Action LoggerAction(LogLevel ll, bool enabled) => - (ll, enabled) switch - { - (LogLevel.Information, true) => (logger, template, parameters) => logger.LogInformation(template, parameters), - (LogLevel.Debug, true) => (logger, template, parameters) => logger.LogDebug(template, parameters), - (LogLevel.Trace, true) => (logger, template, parameters) => logger.LogTrace(template, parameters), - (LogLevel.Warning, true) => (logger, template, parameters) => logger.LogWarning(template, parameters), - (LogLevel.Error, true) => (logger, template, parameters) => logger.LogError(template, parameters), - (LogLevel.Critical, true) => (logger, template, parameters) => logger.LogCritical(template, parameters), - (_, _) => (_, _, _) => { } - }; - } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await options.OuterPipeline.ExecuteAsync(async token => @@ -38,11 +18,11 @@ await options.InnerPipeline.ExecuteAsync(async ct => await using var subscription = new Subscription(); await using var cursor = subscription.Subscribe(options.SubscriberOptions, ct) .GetAsyncEnumerator(ct); - Notify(logger, LogLevel.Information, "{WorkerName} started", WorkerName); + logger.ServiceStarted(WorkerName); while (await cursor.MoveNextAsync().ConfigureAwait(false) && !ct.IsCancellationRequested) - Notify(logger, LogLevel.Trace, "{cursor.Current} processed", cursor.Current); + logger.MessageProcessed(cursor.Current); }, token).ConfigureAwait(false), stoppingToken).ConfigureAwait(false); - Notify(logger, LogLevel.Information, "{WorkerName} stopped", WorkerName); + logger.ServiceStopped(WorkerName); } } From 4741c1aae8b80fed761df01c7a49b16b575d7912 Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 19 Aug 2024 13:36:59 +0200 Subject: [PATCH 76/80] use string interpolation to enable automatic renaming on refactory --- src/UnitTests/subscriber_options_builder.cs | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/UnitTests/subscriber_options_builder.cs b/src/UnitTests/subscriber_options_builder.cs index 7bd0556..540ff29 100644 --- a/src/UnitTests/subscriber_options_builder.cs +++ b/src/UnitTests/subscriber_options_builder.cs @@ -20,7 +20,7 @@ public void requires_at_least_one_method_call_to_connectionstring() { var exception = Record.Exception(() => new OptionsBuilder().Build()); Assert.IsType(exception); - Assert.Equal("`ConnectionString` method not called on OptionsBuilder", exception.Message); + Assert.Equal($"`{nameof(OptionsBuilder.ConnectionString)}` method not called on {nameof(OptionsBuilder)}", exception.Message); } [Fact] @@ -31,7 +31,7 @@ public void requires_at_most_one_method_call_to_connectionstring() .ConnectionString(ValidConnectionString) .Build()); Assert.IsType(exception); - Assert.Equal("`ConnectionString` method on OptionsBuilder called more then once", exception.Message); + Assert.Equal($"`{nameof(OptionsBuilder.ConnectionString)}` method on {nameof(OptionsBuilder)} called more then once", exception.Message); } [Fact] @@ -39,7 +39,7 @@ public void requires_at_least_one_method_call_to_datasource() { var exception = Record.Exception(() => new OptionsBuilder().ConnectionString(ValidConnectionString).Build()); Assert.IsType(exception); - Assert.Equal("`DataSource` method not called on OptionsBuilder", exception.Message); + Assert.Equal($"`{nameof(OptionsBuilder.DataSource)}` method not called on {nameof(OptionsBuilder)}", exception.Message); } [Fact] @@ -47,7 +47,7 @@ public void requires_at_most_one_method_call_to_datasource() { var exception = Record.Exception(() => _builder(ValidConnectionString).DataSource(new NpgsqlDataSourceBuilder(ValidConnectionString).Build()).Build()); Assert.IsType(exception); - Assert.Equal("`DataSource` method on OptionsBuilder called more then once", exception.Message); + Assert.Equal($"`{nameof(OptionsBuilder.DataSource)}` method on {nameof(OptionsBuilder)} called more then once", exception.Message); } @@ -56,7 +56,7 @@ public void requires_at_least_one_method_call_to_consumes() { var exception = Record.Exception(() => _builder(ValidConnectionString).Build()); Assert.IsType(exception); - Assert.Equal("No `Consumes...` method called on OptionsBuilder", exception.Message); + Assert.Equal($"No `Consumes...` method called on {nameof(OptionsBuilder)}", exception.Message); } [Fact] @@ -105,7 +105,7 @@ public void ConsumesRawObjects_cannot_be_mixed_with_other_consuming_strategies() _builder(ValidConnectionString).ConsumesRawStrings(messageHandler2).ConsumesRawObjects(messageHandler1) .Build()); Assert.IsType(exception); - Assert.Equal("`ConsumesRawObjects` cannot be mixed with other consuming strategies", exception.Message); + Assert.Equal($"`{nameof(OptionsBuilder.ConsumesRawObjects)}` cannot be mixed with other consuming strategies", exception.Message); } [Fact] @@ -117,7 +117,7 @@ public void ConsumesRawStrings_cannot_be_mixed_with_other_consuming_strategies() _builder(ValidConnectionString).ConsumesRawObjects(messageHandler1).ConsumesRawStrings(messageHandler2) .Build()); Assert.IsType(exception); - Assert.Equal("`ConsumesRawStrings` cannot be mixed with other consuming strategies", exception.Message); + Assert.Equal($"`{nameof(OptionsBuilder.ConsumesRawStrings)}` cannot be mixed with other consuming strategies", exception.Message); } [Fact] @@ -126,7 +126,7 @@ public void with_typed_raw_consumer_of_object_requires_RawUrn_decoration() var messageHandler = Substitute.For>(); var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawObject(messageHandler).Build()); Assert.IsType(exception); - Assert.Equal($"`{nameof(RawRoutedByUrnAttribute)}` missing on `InvalidMessage` message type", exception.Message); + Assert.Equal($"`{nameof(RawRoutedByUrnAttribute)}` missing on `{nameof(InvalidMessage)}` message type", exception.Message); } [Fact] @@ -135,7 +135,7 @@ public void with_typed_raw_consumer_of_string_requires_RawUrn_decoration() var messageHandler = Substitute.For>(); var exception = Record.Exception(() => _builder(ValidConnectionString).ConsumesRawString(messageHandler).Build()); Assert.IsType(exception); - Assert.Equal($"`{nameof(RawRoutedByUrnAttribute)}` missing on `InvalidMessage` message type", exception.Message); + Assert.Equal($"`{nameof(RawRoutedByUrnAttribute)}` missing on `{nameof(InvalidMessage)}` message type", exception.Message); } [Fact] @@ -151,7 +151,7 @@ public void does_not_allow_multiple_registration_of_the_same_typed_consumer() .AndNamingPolicy(new AttributeNamingPolicy()) .Build()); Assert.IsType(exception); - Assert.Equal("`UserCreatedContract` was already registered.", exception.Message); + Assert.Equal($"`{nameof(UserCreatedContract)}` was already registered.", exception.Message); } [Fact] @@ -168,7 +168,7 @@ public void with_typed_consumer_allows_only_one_naming_policy_instance() .AndNamingPolicy(new AttributeNamingPolicy()) .Build()); Assert.IsType(exception); - Assert.Equal("`NamingPolicy` method on OptionsBuilder called more then once", exception.Message); + Assert.Equal($"`{nameof(OptionsBuilder.NamingPolicy)}` method on OptionsBuilder called more then once", exception.Message); } } } From 321cc8bd23a52afbb48348d67e4410ca674d4c02 Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 19 Aug 2024 14:24:54 +0200 Subject: [PATCH 77/80] use evtension method --- .../Replication/IReplicationDataMapper.cs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs index dfcdf3f..6f7f7e3 100644 --- a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs @@ -19,13 +19,9 @@ internal class ReplicationDataMapper(IDictionary>, IReplicationJsonBMapper> _memoizer = Memoizer>, IReplicationJsonBMapper>.Execute(SelectMapper); - private static IReplicationJsonBMapper SelectMapper(string key, - IDictionary> registry) => - registry.TryGetValue(key, out var tuple) - ? tuple.Item1 - : registry.TryGetValue(OptionsBuilder.WildCard, out tuple) - ? tuple.Item1 - : throw new Exception($"Unexpected message `{key}`"); + private static IReplicationJsonBMapper SelectMapper(string key, IDictionary> registry) + => registry.FindByMultiKey(key, OptionsBuilder.WildCard)?.Item1 + ?? throw new NotSupportedException($"Unexpected message `{key}`"); public async Task ReadFromReplication(InsertMessage insertMessage, CancellationToken ct) { From cea26de2df23207b602f1c396f84731fff5d1544 Mon Sep 17 00:00:00 2001 From: giordanol Date: Mon, 19 Aug 2024 14:25:43 +0200 Subject: [PATCH 78/80] default on not found --- src/Blumchen/IDictionaryExtensions.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Blumchen/IDictionaryExtensions.cs b/src/Blumchen/IDictionaryExtensions.cs index 48fe3f3..999f8a9 100644 --- a/src/Blumchen/IDictionaryExtensions.cs +++ b/src/Blumchen/IDictionaryExtensions.cs @@ -2,7 +2,11 @@ namespace Blumchen; internal static class IDictionaryExtensions { - public static TR FindByMultiKey(this IDictionary registry, params T[] parameters) - where T : class => - !registry.TryGetValue(parameters[0], out var value) ? registry.FindByMultiKey(parameters[1..parameters.Length]) : value; + public static TR? FindByMultiKey(this IDictionary registry, params T[] parameters) where T : class + { + if (parameters.Length == 0) return default; + return registry.TryGetValue(parameters[0], out var value) + ? value + : FindByMultiKey(registry, parameters[1..parameters.Length]); + } } From 0db1a31c8abc8815db89d1ffb6587f28b7cc6211 Mon Sep 17 00:00:00 2001 From: giordanol Date: Tue, 20 Aug 2024 10:31:50 +0200 Subject: [PATCH 79/80] added test clause --- src/Tests/DatabaseFixture.cs | 3 +++ src/Tests/ServiceWorker..cs | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Tests/DatabaseFixture.cs b/src/Tests/DatabaseFixture.cs index eb27253..ba5ee9e 100644 --- a/src/Tests/DatabaseFixture.cs +++ b/src/Tests/DatabaseFixture.cs @@ -37,9 +37,12 @@ public async Task Handle(T value) protected class TestHandler(ILogger> logger): IMessageHandler where T : class { + private int _counter; + internal int Counter => _counter; public Task Handle(T value) { logger.LogTrace($"Message consumed:{value}"); + Interlocked.Increment(ref _counter); return Task.CompletedTask; } } diff --git a/src/Tests/ServiceWorker..cs b/src/Tests/ServiceWorker..cs index 675edd6..9b52684 100644 --- a/src/Tests/ServiceWorker..cs +++ b/src/Tests/ServiceWorker..cs @@ -91,7 +91,7 @@ await MessageAppender.AppendAsync( Container.GetConnectionString(), ct.Token ); - + var builder = Host.CreateApplicationBuilder(); builder.Services .AddXunitLogging(Output) @@ -101,9 +101,10 @@ await MessageAppender.AppendAsync( consumesFn ); - await builder - .Build() - .RunAsync(ct.Token); + using var host = builder.Build(); + var handler = host.Services.GetRequiredService>(); + await host.RunAsync(ct.Token); + Assert.True(handler.Counter > 0); } } From 7968c6dc2dbea0afc740c82b96c52439630f741d Mon Sep 17 00:00:00 2001 From: giordanol Date: Tue, 20 Aug 2024 10:28:24 +0200 Subject: [PATCH 80/80] added ImTools --- src/Blumchen/Blumchen.csproj | 1 + src/Blumchen/Ensure.cs | 41 +++++++++++----- .../Subscriber/OptionsBuilder.Consumes.cs | 9 +++- src/Blumchen/Subscriber/OptionsBuilder.cs | 44 +++++++++++------ src/Blumchen/Subscriber/TypeExtensions.cs | 10 ++++ src/Blumchen/Subscriptions/Memoizer.cs | 38 -------------- .../Replication/IReplicationDataMapper.cs | 33 ++++++++++--- src/Blumchen/Subscriptions/Subscription.cs | 49 ++++++++++--------- 8 files changed, 126 insertions(+), 99 deletions(-) create mode 100644 src/Blumchen/Subscriber/TypeExtensions.cs delete mode 100644 src/Blumchen/Subscriptions/Memoizer.cs diff --git a/src/Blumchen/Blumchen.csproj b/src/Blumchen/Blumchen.csproj index 9b4e0d9..89c220c 100644 --- a/src/Blumchen/Blumchen.csproj +++ b/src/Blumchen/Blumchen.csproj @@ -48,6 +48,7 @@ + all none diff --git a/src/Blumchen/Ensure.cs b/src/Blumchen/Ensure.cs index 75d8562..7ecafb4 100644 --- a/src/Blumchen/Ensure.cs +++ b/src/Blumchen/Ensure.cs @@ -1,32 +1,49 @@ using System.Collections; using Blumchen.Serialization; using Blumchen.Subscriber; +using ImTools; namespace Blumchen; internal static class Ensure { - public static void RawUrn(T value, string parameters) => new RawUrnTrait().IsValid(value, parameters); + internal const string CannotBeMixedWithOtherConsumingStrategies = "`{0}` cannot be mixed with other consuming strategies"; + + internal const string RawRoutedByUrnErrorFormat = $"`{nameof(RawRoutedByUrnAttribute)}` missing on `{{0}}` message type"; + + public static void RawUrn(T value, params object?[] parameters) => new RawUrnTrait().IsValid(value, parameters); public static void Null(T value, string parameters) => new NullTrait().IsValid(value, parameters); public static void NotNull(T value, string parameters) => new NotNullTrait().IsValid(value, parameters); public static void NotEmpty(T value, string parameters) => new NotEmptyTrait().IsValid(value, parameters); public static void Empty(T value, string parameters) => new EmptyTrait().IsValid(value, parameters); - public static bool Empty(T value1, TU value2, params string[] parameters) => - new EmptyTrait().IsValid(value1, parameters) && new EmptyTrait().IsValid(value2, parameters); + public static void And(Validable left, T v1, Validable right, TU v2, params object?[] parameters) => + _ = left.IsValid(v1, parameters) && right.IsValid(v2, parameters); } -internal abstract class Validable(Func condition, string errorFormat) +internal abstract record Validable(Func Condition, string ErrorFormat) { - public bool IsValid(T value, params string[] parameters) + public bool IsValid(T value, params object?[] parameters) { - if (!condition(value)) - throw new ConfigurationException(string.Format(errorFormat, parameters)); + if (!Condition(value)) + throw new ConfigurationException(string.Format(ErrorFormat, parameters)); return true; } } -internal class RawUrnTrait(): Validable(v => v is ICollection { Count: > 0 }, $"`{nameof(RawRoutedByUrnAttribute)}` missing on `{typeof(TR).Name}` message type"); -internal class NullTrait(): Validable(v => v is null, $"`{{0}}` method on {nameof(OptionsBuilder)} called more then once"); -internal class NotNullTrait(): Validable(v => v is not null, $"`{{0}}` method not called on {nameof(OptionsBuilder)}"); -internal class NotEmptyTrait(): Validable(v => v is ICollection { Count: > 0 }, $"No `{{0}}` method called on {nameof(OptionsBuilder)}"); -internal class EmptyTrait(): Validable(v => v is ICollection { Count: 0 }, $"`{{0}}` cannot be mixed with other consuming strategies"); +internal record RawUrnTrait(): Validable(v => v is ICollection { Count: > 0 }, + Ensure.RawRoutedByUrnErrorFormat); + +internal record NullTrait() + : Validable(v => v is null, $"`{{0}}` method on {nameof(OptionsBuilder)} called more then once"); + +internal record NotNullTrait() + : Validable(v => v is not null, $"`{{0}}` method not called on {nameof(OptionsBuilder)}"); + +internal record NotEmptyTrait(): Validable(v => v is ICollection { Count: > 0 }, + $"No `{{0}}` method called on {nameof(OptionsBuilder)}"); + +internal record EmptyTrait() + : Validable(v => v is ICollection { Count: 0 }, Ensure.CannotBeMixedWithOtherConsumingStrategies); + +internal record BoolTrait(Func Condition, string ErrorFormat) + : Validable(Condition, ErrorFormat); diff --git a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs index f5035c5..2e64319 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.Consumes.cs @@ -2,6 +2,7 @@ using System.Text.Json.Serialization; using Blumchen.Serialization; using Blumchen.Subscriptions.Replication; +using ImTools; using JetBrains.Annotations; namespace Blumchen.Subscriber; @@ -58,9 +59,13 @@ internal IConsumesTypedJsonOptionsContext Consumes(IMessageHandler handler .GetMethod(nameof(IMessageHandler.Handle), BindingFlags.Instance | BindingFlags.Public, [typeof(T)]) ?? throw new ConfigurationException($"Unable to find {nameof(IMessageHandler)} implementation on {handler.GetType().Name}"); - if (_typeRegistry.ContainsKey(typeof(T))) + if (_typeRegistry.GetEntryOrNull(typeof(T).GetHashCode()) != null) throw new ConfigurationException($"`{typeof(T).Name}` was already registered."); - _typeRegistry.Add(typeof(T), new Tuple(handler, methodInfo)); + _typeRegistry = _typeRegistry.AddSureNotPresentEntry( + new KVEntry>(typeof(T).GetHashCode(), typeof(T), + new Tuple(handler, methodInfo)) + ); + return new ConsumesTypedJsonTypedJsonOptionsContext(this); } diff --git a/src/Blumchen/Subscriber/OptionsBuilder.cs b/src/Blumchen/Subscriber/OptionsBuilder.cs index 01c7a13..2c47388 100644 --- a/src/Blumchen/Subscriber/OptionsBuilder.cs +++ b/src/Blumchen/Subscriber/OptionsBuilder.cs @@ -1,11 +1,15 @@ +using System.Collections; using System.Reflection; +using System.Security.Principal; using System.Text.Json.Serialization; using Blumchen.Serialization; using Blumchen.Subscriptions; using Blumchen.Subscriptions.Management; using Blumchen.Subscriptions.Replication; +using ImTools; using JetBrains.Annotations; using Npgsql; +// ReSharper disable RedundantDefaultMemberInitializer namespace Blumchen.Subscriber; @@ -22,7 +26,7 @@ public sealed partial class OptionsBuilder private PublicationManagement.PublicationOptions _publicationOptions = new(); private ReplicationSlotManagement.ReplicationSlotOptions? _replicationSlotOptions; - private readonly Dictionary> _typeRegistry = []; + private ImHashMap> _typeRegistry = ImHashMap>.Empty; private readonly Dictionary> _replicationDataMapperSelector = []; @@ -86,7 +90,11 @@ public OptionsBuilder ConsumesRawString(IMessageHandler handler) wher [UsedImplicitly] public OptionsBuilder ConsumesRawStrings(IMessageHandler handler) { - Ensure.Empty(_replicationDataMapperSelector, _typeRegistry, nameof(ConsumesRawStrings)); + Ensure.And( + new EmptyTrait(), _replicationDataMapperSelector, + new BoolTrait>>(r => r.IsEmpty, Ensure.CannotBeMixedWithOtherConsumingStrategies), + _typeRegistry,nameof(ConsumesRawStrings) + ); var methodInfo = handler .GetType() @@ -101,7 +109,11 @@ public OptionsBuilder ConsumesRawStrings(IMessageHandler handler) [UsedImplicitly] public OptionsBuilder ConsumesRawObjects(IMessageHandler handler) { - Ensure.Empty(_replicationDataMapperSelector, _typeRegistry, nameof(ConsumesRawObjects)); + Ensure.And( + new EmptyTrait(), _replicationDataMapperSelector, + new BoolTrait>>(r => r.IsEmpty, Ensure.CannotBeMixedWithOtherConsumingStrategies), + _typeRegistry, nameof(ConsumesRawObjects) + ); var methodInfo = handler .GetType() @@ -117,22 +129,22 @@ public OptionsBuilder ConsumesRawObjects(IMessageHandler handler) private OptionsBuilder ConsumesRaw(IMessageHandler handler, IReplicationJsonBMapper dataMapper) where T : class where TU : class { - var urns = typeof(T) - .GetCustomAttributes(typeof(RawRoutedByUrnAttribute), false) - .Union(typeof(T) - .GetCustomAttributes(typeof(RawRoutedByStringAttribute), false)) - .OfType() - .Select(attribute => attribute.Route).ToList(); - Ensure.RawUrn,T>(urns, nameof(NamingPolicy)); + var routingList = typeof(T) + .GetAttributes() + .Union( + typeof(T).GetAttributes() + ) + .Select(routed => routed.Route).ToList(); + Ensure.RawUrn>(routingList, typeof(T).Name); var methodInfo = handler .GetType() .GetMethod(nameof(IMessageHandler.Handle), BindingFlags.Instance | BindingFlags.Public, [typeof(TU)]) ?? throw new ConfigurationException($"Unable to find {nameof(IMessageHandler)} implementation on {handler.GetType().Name}"); - using var urnEnum = urns.GetEnumerator(); + using var urnEnum = routingList.GetEnumerator(); while (urnEnum.MoveNext()) - _replicationDataMapperSelector.Add(urnEnum.Current.ToString(), + _replicationDataMapperSelector.Add(urnEnum.Current, new Tuple(dataMapper, handler, methodInfo)); return this; } @@ -150,19 +162,19 @@ internal ISubscriberOptions Build() Ensure.NotNull(_connectionStringBuilder, $"{nameof(ConnectionString)}"); Ensure.NotNull(_dataSource, $"{nameof(DataSource)}"); - if (_typeRegistry.Count > 0) + if (!_typeRegistry.IsEmpty) { Ensure.NotNull(_namingPolicy, $"{nameof(NamingPolicy)}"); if (_jsonSerializerContext != null) { var typeResolver = new JsonTypeResolver(_jsonSerializerContext, _namingPolicy); - foreach (var type in _typeRegistry.Keys) - typeResolver.WhiteList(type); + foreach (var kv in _typeRegistry.Enumerate()) + typeResolver.WhiteList(kv.Key); _jsonDataMapper = new JsonReplicationDataMapper(typeResolver, new JsonReplicationDataReader(typeResolver)); - foreach (var (key, value) in typeResolver.RegisteredTypes.Join(_typeRegistry, pair => pair.Value, + foreach (var (key, value) in typeResolver.RegisteredTypes.Join(_typeRegistry.Enumerate(), pair => pair.Value, pair => pair.Key, (pair, valuePair) => (pair.Key, valuePair.Value))) _replicationDataMapperSelector.Add(key, new Tuple(_jsonDataMapper, value.Item1, value.Item2)); diff --git a/src/Blumchen/Subscriber/TypeExtensions.cs b/src/Blumchen/Subscriber/TypeExtensions.cs new file mode 100644 index 0000000..2dcd3e8 --- /dev/null +++ b/src/Blumchen/Subscriber/TypeExtensions.cs @@ -0,0 +1,10 @@ +using Blumchen.Serialization; + +namespace Blumchen.Subscriber; + +internal static class TypeExtensions +{ + public static IEnumerable GetAttributes(this Type type) + where T : Attribute, IRouted => + type.GetCustomAttributes(typeof(T), false).OfType(); +} diff --git a/src/Blumchen/Subscriptions/Memoizer.cs b/src/Blumchen/Subscriptions/Memoizer.cs deleted file mode 100644 index 22e27dd..0000000 --- a/src/Blumchen/Subscriptions/Memoizer.cs +++ /dev/null @@ -1,38 +0,0 @@ -namespace Blumchen.Subscriptions; - -internal class Memoizer(Func func) - where T : notnull -{ - private readonly Dictionary _memoizer = new(); - - private TR Caller(T value1, TU value2) - { - if (_memoizer.TryGetValue(value1, out var result)) - return result; - return _memoizer[value1] = func(value1, value2); - } - - public static Func Execute(Func func) - { - var memoizer = new Memoizer(func); - return (x, y) => memoizer.Caller(x, y); - } -} -internal class Memoizer(Func func) - where T : notnull -{ - private readonly Dictionary _memoizer = new(); - - private TT Caller(T value1, TU value2, TR value3) - { - if (_memoizer.TryGetValue(value1, out var result)) - return result; - return _memoizer[value1] = func(value1, value2, value3); - } - - public static Func Execute(Func func) - { - var memoizer = new Memoizer(func); - return (x, y,z) => memoizer.Caller(x, y,z); - } -} diff --git a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs index 6f7f7e3..570c08a 100644 --- a/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs +++ b/src/Blumchen/Subscriptions/Replication/IReplicationDataMapper.cs @@ -4,6 +4,7 @@ using Npgsql.Replication.PgOutput.Messages; using System.Text.Json; using Blumchen.Subscriber; +using ImTools; namespace Blumchen.Subscriptions.Replication; @@ -17,11 +18,12 @@ public interface IReplicationDataMapper internal class ReplicationDataMapper(IDictionary> mapperSelector) : IReplicationDataMapper { - private readonly Func>, IReplicationJsonBMapper> _memoizer = Memoizer>, IReplicationJsonBMapper>.Execute(SelectMapper); - private static IReplicationJsonBMapper SelectMapper(string key, IDictionary> registry) - => registry.FindByMultiKey(key, OptionsBuilder.WildCard)?.Item1 - ?? throw new NotSupportedException($"Unexpected message `{key}`"); + private ImHashMap _memo = ImHashMap.Empty; + + private static IReplicationJsonBMapper SelectMapper(string key, IDictionary> registry) => + registry.FindByMultiKey(key, OptionsBuilder.WildCard)?.Item1 + ?? throw new NotSupportedException($"Unexpected message `{key}`"); public async Task ReadFromReplication(InsertMessage insertMessage, CancellationToken ct) { @@ -47,7 +49,13 @@ public async Task ReadFromReplication(InsertMessage insertMessage, Ca } case 2 when column.GetDataTypeName().Equals("jsonb", StringComparison.OrdinalIgnoreCase): - return await _memoizer(typeName, mapperSelector).ReadFromReplication(id, typeName, column, ct); + + _memo = _memo.AddOrGetEntry(typeName.GetHashCode(), + new KVEntry(typeName.GetHashCode(), typeName, + SelectMapper(typeName, mapperSelector))); + return await _memo.GetValueOrDefault(typeName.GetHashCode(), typeName) + .ReadFromReplication(id, typeName, column, ct); + } } catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException @@ -62,6 +70,14 @@ public async Task ReadFromReplication(InsertMessage insertMessage, Ca throw new InvalidOperationException("You should not get here"); } + private IReplicationJsonBMapper Get(string typeName) + { + if (_memo.TryFind(typeName, out var mapper)) return mapper; + mapper = SelectMapper(typeName, mapperSelector); + _memo = _memo.AddOrUpdate(typeName, mapper); + return mapper; + } + public async Task ReadFromSnapshot(NpgsqlDataReader reader, CancellationToken ct) { long id = default; @@ -69,12 +85,13 @@ public async Task ReadFromSnapshot(NpgsqlDataReader reader, Cancellat { id = reader.GetInt64(0); var typeName = reader.GetString(1); - return await _memoizer(typeName, mapperSelector).ReadFromSnapshot(typeName, id, reader, ct); + var mapper = Get(typeName); + return await mapper.ReadFromSnapshot(typeName, id, reader, ct); } - catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or JsonException) + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException + or JsonException) { return new KoEnvelope(ex, id.ToString()); } } } - diff --git a/src/Blumchen/Subscriptions/Subscription.cs b/src/Blumchen/Subscriptions/Subscription.cs index 80c79cf..8d60a5a 100644 --- a/src/Blumchen/Subscriptions/Subscription.cs +++ b/src/Blumchen/Subscriptions/Subscription.cs @@ -1,9 +1,11 @@ +using System.Dynamic; using System.Reflection; using System.Runtime.CompilerServices; using Blumchen.Database; using Blumchen.Subscriber; using Blumchen.Subscriptions.Management; using Blumchen.Subscriptions.Replication; +using ImTools; using Npgsql; using Npgsql.Replication; using Npgsql.Replication.PgOutput; @@ -17,27 +19,7 @@ public sealed class Subscription: IAsyncDisposable private LogicalReplicationConnection? _connection; private readonly OptionsBuilder _builder = new(); - private readonly - Func>, Type, ( - IMessageHandler messageHandler, MethodInfo methodInfo)> _messageHandler; - - public Subscription() - { - _messageHandler = - Memoizer>, Type, ( - IMessageHandler messageHandler, - MethodInfo methodInfo)>.Execute(MessageHandler); - } - - private (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler( - string messageType, IDictionary> registry, - Type objType) - { - var (_, messageHandler, methodInfo) = registry.FindByMultiKey(messageType, OptionsBuilder.WildCard) ?? - throw new NotSupportedException( - $"Unregistered type for {objType.AssemblyQualifiedName}"); - return (messageHandler, methodInfo); - } + private ImHashMap _messageHandlers = ImHashMap.Empty; public enum CreateStyle { @@ -116,6 +98,28 @@ internal async IAsyncEnumerable Subscribe( } } + private (IMessageHandler, MethodInfo) GetConsumer( + string messageType, + Type type, + IDictionary> registry + ) + { + if (_messageHandlers.TryFind(messageType, out var tuple)) return tuple; + tuple = MessageHandler(messageType, registry, type); + _messageHandlers = _messageHandlers.AddOrUpdate(messageType, tuple); + return tuple; + + static (IMessageHandler messageHandler, MethodInfo methodInfo) MessageHandler( + string messageType, IDictionary> registry, + Type objType) + { + var (_, messageHandler, methodInfo) = registry.FindByMultiKey(messageType, OptionsBuilder.WildCard) ?? + throw new NotSupportedException( + $"Unregistered type for {objType.AssemblyQualifiedName}"); + return (messageHandler, methodInfo); + } + } + private async IAsyncEnumerable ProcessEnvelope( IEnvelope envelope, IDictionary> registry, @@ -129,8 +133,7 @@ IErrorProcessor errorProcessor yield break; case OkEnvelope(var value, var messageType): { - var (messageHandler, methodInfo) = - _messageHandler(messageType, registry, value.GetType()); + var (messageHandler, methodInfo) = GetConsumer(messageType, value.GetType(), registry); await ((Task)methodInfo.Invoke(messageHandler, [value])!).ConfigureAwait(false); yield return envelope; yield break;