From 0e756208e8528654b0b2119fb94e322fb16b8fb6 Mon Sep 17 00:00:00 2001 From: Kyle Spearrin Date: Tue, 26 Feb 2019 11:31:37 -0500 Subject: [PATCH] port events processor over to webjobs sdk 3 --- bitwarden-server.sln | 14 +++---- src/EventsProcessor/EventsProcessor.csproj | 18 +++++--- src/EventsProcessor/Functions.cs | 29 ++++++++----- src/EventsProcessor/Program.cs | 41 ++++++++++++++----- .../Properties/webjob-publish-settings.json | 5 --- src/EventsProcessor/appsettings.json | 3 ++ 6 files changed, 71 insertions(+), 39 deletions(-) delete mode 100644 src/EventsProcessor/Properties/webjob-publish-settings.json create mode 100644 src/EventsProcessor/appsettings.json diff --git a/bitwarden-server.sln b/bitwarden-server.sln index becd58e82..54f76fec8 100644 --- a/bitwarden-server.sln +++ b/bitwarden-server.sln @@ -39,14 +39,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Icons", "src\Icons\Icons.cs EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Events", "src\Events\Events.csproj", "{994DD611-F266-4BD3-8072-3B1B57267ED5}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EventsProcessor", "src\EventsProcessor\EventsProcessor.csproj", "{2235D24F-E607-47F4-81AD-BB4504ADF9C6}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Admin", "src\Admin\Admin.csproj", "{B131CEF3-89FB-4C90-ADB0-9E9C4246EB56}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Notifications", "src\Notifications\Notifications.csproj", "{28635027-20E5-42FA-B218-B6C878DE5350}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Core.Test", "test\Core.Test\Core.Test.csproj", "{8EF31E6C-400A-4174-8BE3-502B08FB10B5}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EventsProcessor", "src\EventsProcessor\EventsProcessor.csproj", "{79BB453F-D0D8-4DDF-9809-A405C56692BD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -91,10 +91,6 @@ Global {994DD611-F266-4BD3-8072-3B1B57267ED5}.Debug|Any CPU.Build.0 = Debug|Any CPU {994DD611-F266-4BD3-8072-3B1B57267ED5}.Release|Any CPU.ActiveCfg = Release|Any CPU {994DD611-F266-4BD3-8072-3B1B57267ED5}.Release|Any CPU.Build.0 = Release|Any CPU - {2235D24F-E607-47F4-81AD-BB4504ADF9C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2235D24F-E607-47F4-81AD-BB4504ADF9C6}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2235D24F-E607-47F4-81AD-BB4504ADF9C6}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2235D24F-E607-47F4-81AD-BB4504ADF9C6}.Release|Any CPU.Build.0 = Release|Any CPU {B131CEF3-89FB-4C90-ADB0-9E9C4246EB56}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B131CEF3-89FB-4C90-ADB0-9E9C4246EB56}.Debug|Any CPU.Build.0 = Debug|Any CPU {B131CEF3-89FB-4C90-ADB0-9E9C4246EB56}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -107,6 +103,10 @@ Global {8EF31E6C-400A-4174-8BE3-502B08FB10B5}.Debug|Any CPU.Build.0 = Debug|Any CPU {8EF31E6C-400A-4174-8BE3-502B08FB10B5}.Release|Any CPU.ActiveCfg = Release|Any CPU {8EF31E6C-400A-4174-8BE3-502B08FB10B5}.Release|Any CPU.Build.0 = Release|Any CPU + {79BB453F-D0D8-4DDF-9809-A405C56692BD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {79BB453F-D0D8-4DDF-9809-A405C56692BD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {79BB453F-D0D8-4DDF-9809-A405C56692BD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {79BB453F-D0D8-4DDF-9809-A405C56692BD}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -121,10 +121,10 @@ Global {66B0A682-658A-4A82-B606-A077A4871448} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84E} {9CF59342-3912-4B45-A2BA-0F173666586D} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84D} {994DD611-F266-4BD3-8072-3B1B57267ED5} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84D} - {2235D24F-E607-47F4-81AD-BB4504ADF9C6} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84D} {B131CEF3-89FB-4C90-ADB0-9E9C4246EB56} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84D} {28635027-20E5-42FA-B218-B6C878DE5350} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84D} {8EF31E6C-400A-4174-8BE3-502B08FB10B5} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84F} + {79BB453F-D0D8-4DDF-9809-A405C56692BD} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E01CBF68-2E20-425F-9EDB-E0A6510CA92F} diff --git a/src/EventsProcessor/EventsProcessor.csproj b/src/EventsProcessor/EventsProcessor.csproj index 8a6e7f6db..3753e7fc3 100644 --- a/src/EventsProcessor/EventsProcessor.csproj +++ b/src/EventsProcessor/EventsProcessor.csproj @@ -1,18 +1,26 @@ - + Exe - net471 + netcoreapp2.1 Bit.EventsProcessor - + + + + - - + + + + + PreserveNewest + PreserveNewest + diff --git a/src/EventsProcessor/Functions.cs b/src/EventsProcessor/Functions.cs index 81491a38c..bd837a969 100644 --- a/src/EventsProcessor/Functions.cs +++ b/src/EventsProcessor/Functions.cs @@ -1,13 +1,13 @@ using System; using System.Collections.Generic; -using System.Configuration; -using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Bit.Core.Models.Data; using Bit.Core.Services; using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -15,22 +15,22 @@ namespace Bit.EventsProcessor { public class Functions { - private static IEventWriteService _eventWriteService; + private readonly IEventWriteService _eventWriteService; - static Functions() + public Functions(IConfiguration config) { - var storageConnectionString = ConfigurationManager.ConnectionStrings["AzureWebJobsStorage"]; - if(storageConnectionString == null || string.IsNullOrWhiteSpace(storageConnectionString.ConnectionString)) + var storageConnectionString = config["AzureWebJobsStorage"]; + if(string.IsNullOrWhiteSpace(storageConnectionString)) { return; } - var repo = new Core.Repositories.TableStorage.EventRepository(storageConnectionString.ConnectionString); + var repo = new Core.Repositories.TableStorage.EventRepository(storageConnectionString); _eventWriteService = new RepositoryEventWriteService(repo); } - public async static Task ProcessQueueMessageAsync([QueueTrigger("event")] string message, - TextWriter logger, CancellationToken cancellationToken) + public async Task ProcessQueueMessageAsync([QueueTrigger("event")] string message, + CancellationToken cancellationToken, ILogger logger) { if(_eventWriteService == null || message == null || message.Length == 0) { @@ -39,6 +39,7 @@ namespace Bit.EventsProcessor try { + logger.LogInformation("Processing message."); var events = new List(); var token = JToken.Parse(message); @@ -55,14 +56,20 @@ namespace Bit.EventsProcessor } await _eventWriteService.CreateManyAsync(events); + logger.LogInformation("Processed message."); } catch(JsonReaderException) { - await logger.WriteLineAsync("JsonReaderException: Unable to parse message."); + logger.LogError("JsonReaderException: Unable to parse message."); } catch(JsonSerializationException) { - await logger.WriteLineAsync("JsonSerializationException: Unable to serialize token."); + logger.LogError("JsonSerializationException: Unable to serialize token."); + } + catch(Exception e) + { + logger.LogError(e, "Exception occurred. " + e.Message); + throw e; } } } diff --git a/src/EventsProcessor/Program.cs b/src/EventsProcessor/Program.cs index e4c5e02f9..fb8266e50 100644 --- a/src/EventsProcessor/Program.cs +++ b/src/EventsProcessor/Program.cs @@ -1,21 +1,40 @@ -using Microsoft.Azure.WebJobs; +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; namespace Bit.EventsProcessor { - public class Program + class Program { - private static void Main() + static void Main(string[] args) { - var config = new JobHostConfiguration(); - if(config.IsDevelopment) + var builder = new HostBuilder(); + builder.ConfigureWebJobs(b => { - config.UseDevelopmentSettings(); + b.AddAzureStorageCoreServices(); + b.AddAzureStorage(a => + { + a.BatchSize = 5; + }); + // Not working. ref: https://github.com/Azure/azure-webjobs-sdk/issues/1962 + b.AddDashboardLogging(); + }); + builder.ConfigureLogging((context, b) => + { + b.AddConsole(); + b.SetMinimumLevel(LogLevel.Warning); + }); + builder.ConfigureHostConfiguration(b => + { + b.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true); + b.AddEnvironmentVariables(); + }); + var host = builder.Build(); + using(host) + { + host.Run(); } - - config.Queues.BatchSize = 5; - - var host = new JobHost(config); - host.RunAndBlock(); } } } diff --git a/src/EventsProcessor/Properties/webjob-publish-settings.json b/src/EventsProcessor/Properties/webjob-publish-settings.json deleted file mode 100644 index 17bf2837e..000000000 --- a/src/EventsProcessor/Properties/webjob-publish-settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "$schema": "http://schemastore.org/schemas/json/webjob-publish-settings.json", - "webJobName": "EventsProcessor", - "runMode": "Continuous" -} \ No newline at end of file diff --git a/src/EventsProcessor/appsettings.json b/src/EventsProcessor/appsettings.json new file mode 100644 index 000000000..32dd707c6 --- /dev/null +++ b/src/EventsProcessor/appsettings.json @@ -0,0 +1,3 @@ +{ + "AzureWebJobsStorage": "" +}