Upstream/bitwarden-server
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2024-12-24 17:17:40 +01:00
Code Issues Projects Releases Wiki Activity

move event processor to aspnet hosted service

Browse Source
This commit is contained in:
Kyle Spearrin 2019-03-02 12:15:28 -05:00
parent c4ac86d4f4
commit 66729fec3f
9 changed files with 216 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
src/EventsProcessor/AzureQueueHostedService.cs Normal file
View File

@ -0,0 +1,134 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Bit.Core.Models.Data;
using Bit.Core.Services;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Queue;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace Bit.EventsProcessor
{
public class AzureQueueHostedService : IHostedService, IDisposable
{
private readonly ILogger<AzureQueueHostedService> _logger;
private readonly IConfiguration _configuration;
private Task _executingTask;
private CancellationTokenSource _cts;
private CloudQueue _queue;
private IEventWriteService _eventWriteService;
public AzureQueueHostedService(
ILogger<AzureQueueHostedService> logger,
IConfiguration configuration)
{
_logger = logger;
_configuration = configuration;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
_executingTask = ExecuteAsync(_cts.Token);
return _executingTask.IsCompleted ? _executingTask : Task.CompletedTask;
}
public async Task StopAsync(CancellationToken cancellationToken)
{
if(_executingTask == null)
{
return;
}
_cts.Cancel();
await Task.WhenAny(_executingTask, Task.Delay(-1, cancellationToken));
cancellationToken.ThrowIfCancellationRequested();
}
public void Dispose()
{ }
private async Task ExecuteAsync(CancellationToken cancellationToken)
{
var storageConnectionString = _configuration["azureStorageConnectionString"];
if(string.IsNullOrWhiteSpace(storageConnectionString))
{
return;
}
var repo = new Core.Repositories.TableStorage.EventRepository(storageConnectionString);
_eventWriteService = new RepositoryEventWriteService(repo);
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var queueClient = storageAccount.CreateCloudQueueClient();
_queue = queueClient.GetQueueReference("event");
while(!cancellationToken.IsCancellationRequested)
{
var messages = await _queue.GetMessagesAsync(32, TimeSpan.FromMinutes(1),
null, null, cancellationToken);
if(messages.Any())
{
foreach(var message in messages)
{
await ProcessQueueMessageAsync(message.AsString, cancellationToken);
await _queue.DeleteMessageAsync(message);
}
}
else
{
await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
}
}
}
public async Task ProcessQueueMessageAsync(string message, CancellationToken cancellationToken)
{
if(_eventWriteService == null || message == null || message.Length == 0)
{
return;
}
try
{
_logger.LogInformation("Processing message.");
var events = new List<IEvent>();
var token = JToken.Parse(message);
if(token is JArray)
{
var indexedEntities = token.ToObject<List<EventMessage>>()
.SelectMany(e => EventTableEntity.IndexEvent(e));
events.AddRange(indexedEntities);
}
else if(token is JObject)
{
var eventMessage = token.ToObject<EventMessage>();
events.AddRange(EventTableEntity.IndexEvent(eventMessage));
}
await _eventWriteService.CreateManyAsync(events);
_logger.LogInformation("Processed message.");
}
catch(JsonReaderException)
{
_logger.LogError("JsonReaderException: Unable to parse message.");
}
catch(JsonSerializationException)
{
_logger.LogError("JsonSerializationException: Unable to serialize token.");
}
catch(Exception e)
{
_logger.LogError(e, "Exception occurred. " + e.Message);
throw e;
}
}
}
}

30
src/EventsProcessor/EventsProcessor.csproj
View File

@ -1,30 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk;Microsoft.NET.Sdk.Publish">
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<OutputType>Exe</OutputType>
<Version>1.30.0</Version>
<TargetFramework>netcoreapp2.1</TargetFramework>
<RootNamespace>Bit.EventsProcessor</RootNamespace>
<UserSecretsId>bitwarden-EventsProcessor</UserSecretsId>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.4" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="3.0.1" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="3.0.3" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.App" Version="2.1.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Core\Core.csproj" />
</ItemGroup>
<ItemGroup>
<None Remove="appsettings.json" />
</ItemGroup>
<ItemGroup>
<Content Include="appsettings.json">
<CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<None Update="Settings.job">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>
</Project>

76
src/EventsProcessor/Functions.cs
View File

@ -1,76 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Bit.Core.Models.Data;
using Bit.Core.Services;
using Microsoft.Azure.WebJobs;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace Bit.EventsProcessor
{
public class Functions
{
private readonly IEventWriteService _eventWriteService;
public Functions(IConfiguration config)
{
var storageConnectionString = config["AzureWebJobsStorage"];
if(string.IsNullOrWhiteSpace(storageConnectionString))
{
return;
}
var repo = new Core.Repositories.TableStorage.EventRepository(storageConnectionString);
_eventWriteService = new RepositoryEventWriteService(repo);
}
public async Task ProcessQueueMessageAsync([QueueTrigger("event")] string message,
CancellationToken cancellationToken, ILogger logger)
{
if(_eventWriteService == null || message == null || message.Length == 0)
{
return;
}
try
{
logger.LogInformation("Processing message.");
var events = new List<IEvent>();
var token = JToken.Parse(message);
if(token is JArray)
{
var indexedEntities = token.ToObject<List<EventMessage>>()
.SelectMany(e => EventTableEntity.IndexEvent(e));
events.AddRange(indexedEntities);
}
else if(token is JObject)
{
var eventMessage = token.ToObject<EventMessage>();
events.AddRange(EventTableEntity.IndexEvent(eventMessage));
}
await _eventWriteService.CreateManyAsync(events);
logger.LogInformation("Processed message.");
}
catch(JsonReaderException)
{
logger.LogError("JsonReaderException: Unable to parse message.");
}
catch(JsonSerializationException)
{
logger.LogError("JsonSerializationException: Unable to serialize token.");
}
catch(Exception e)
{
logger.LogError(e, "Exception occurred. " + e.Message);
throw e;
}
}
}
}

41
src/EventsProcessor/Program.cs
View File

@ -1,40 +1,17 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.AspNetCore;
using Microsoft.AspNetCore.Hosting;
namespace Bit.EventsProcessor
{
class Program
public class Program
{
static void Main(string[] args)
public static void Main(string[] args)
{
var builder = new HostBuilder();
builder.ConfigureWebJobs(b =>
{
b.AddAzureStorageCoreServices();
b.AddAzureStorage(a =>
{
a.BatchSize = 5;
});
// Not working. ref: https://github.com/Azure/azure-webjobs-sdk/issues/1962
b.AddDashboardLogging();
});
builder.ConfigureLogging((context, b) =>
{
b.AddConsole();
b.SetMinimumLevel(LogLevel.Warning);
});
builder.ConfigureHostConfiguration(b =>
{
b.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
b.AddEnvironmentVariables();
});
var host = builder.Build();
using(host)
{
host.Run();
}
WebHost
.CreateDefaultBuilder(args)
.UseStartup<Startup>()
.Build()
.Run();
}
}
}

27
src/EventsProcessor/Properties/launchSettings.json Normal file
View File

@ -0,0 +1,27 @@
{
"iisSettings": {
"windowsAuthentication": false,
"anonymousAuthentication": true,
"iisExpress": {
"applicationUrl": "http://localhost:54103/",
"sslPort": 0
}
},
"profiles": {
"IIS Express": {
"commandName": "IISExpress",
"launchBrowser": true,
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"EventsProcessor": {
"commandName": "Project",
"launchBrowser": true,
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
},
"applicationUrl": "http://localhost:54104/"
}
}
}

22
src/EventsProcessor/Settings.job
View File

@ -1,22 +0,0 @@
{
// Examples:
// Runs every minute
// "schedule": "0 * * * * *"
// Runs every 15 minutes
// "schedule": "0 */15 * * * *"
// Runs every hour (i.e. whenever the count of minutes is 0)
// "schedule": "0 0 * * * *"
// Runs every hour from 9 AM to 5 PM
// "schedule": "0 0 9-17 * * *"
// Runs at 9:30 AM every day
// "schedule": "0 30 9 * * *"
// Runs at 9:30 AM every week day
// "schedule": "0 30 9 * * 1-5"
}

34
src/EventsProcessor/Startup.cs Normal file
View File

@ -0,0 +1,34 @@
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.IdentityModel.Logging;
namespace Bit.EventsProcessor
{
public class Startup
{
public Startup(IHostingEnvironment env, IConfiguration configuration)
{
Configuration = configuration;
Environment = env;
}
public IConfiguration Configuration { get; }
public IHostingEnvironment Environment { get; set; }
public void ConfigureServices(IServiceCollection services)
{
services.AddOptions();
services.AddHostedService<AzureQueueHostedService>();
}
public void Configure(IApplicationBuilder app, IHostingEnvironment env)
{
if(env.IsDevelopment())
{
IdentityModelEventSource.ShowPII = true;
}
}
}
}

2
src/EventsProcessor/appsettings.Production.json Normal file
View File

@ -0,0 +1,2 @@
{
}

2
src/EventsProcessor/appsettings.json
View File

@ -1,3 +1,3 @@
{
"AzureWebJobsStorage": ""
"azureStorageConnectionString": "SECRET"
}