From 8427c23b5eabb42559b0411d39d529409bb314c3 Mon Sep 17 00:00:00 2001 From: Kyle Spearrin Date: Mon, 18 Mar 2019 16:23:37 -0400 Subject: [PATCH] amazon sqs block ip queuing --- .../AmazonSqsBlockIpHostedService.cs | 89 +++++++++++++++++++ .../AzureQueueBlockIpHostedService.cs | 74 +++++++++++++++ .../HostedServices/BlockIpHostedService.cs | 80 +++-------------- src/Admin/Startup.cs | 9 +- src/Core/Core.csproj | 1 + .../AmazonSqsBlockIpService.cs | 72 +++++++++++++++ .../Utilities/ServiceCollectionExtensions.cs | 4 + 7 files changed, 262 insertions(+), 67 deletions(-) create mode 100644 src/Admin/HostedServices/AmazonSqsBlockIpHostedService.cs create mode 100644 src/Admin/HostedServices/AzureQueueBlockIpHostedService.cs create mode 100644 src/Core/Services/Implementations/AmazonSqsBlockIpService.cs diff --git a/src/Admin/HostedServices/AmazonSqsBlockIpHostedService.cs b/src/Admin/HostedServices/AmazonSqsBlockIpHostedService.cs new file mode 100644 index 000000000..850b1b19e --- /dev/null +++ b/src/Admin/HostedServices/AmazonSqsBlockIpHostedService.cs @@ -0,0 +1,89 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Amazon; +using Amazon.SQS; +using Amazon.SQS.Model; +using Bit.Core; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Bit.Admin.HostedServices +{ + public class AmazonSqsBlockIpHostedService : BlockIpHostedService + { + private AmazonSQSClient _client; + + public AmazonSqsBlockIpHostedService( + ILogger logger, + IOptions adminSettings, + GlobalSettings globalSettings) + : base(logger, adminSettings, globalSettings) + { } + + public override void Dispose() + { + _client?.Dispose(); + } + + protected override async Task ExecuteAsync(CancellationToken cancellationToken) + { + _client = new AmazonSQSClient(_globalSettings.Amazon.AccessKeyId, + _globalSettings.Amazon.AccessKeySecret, RegionEndpoint.GetBySystemName(_globalSettings.Amazon.Region)); + var blockIpQueue = await _client.GetQueueUrlAsync("block-ip", cancellationToken); + var blockIpQueueUrl = blockIpQueue.QueueUrl; + var unblockIpQueue = await _client.GetQueueUrlAsync("unblock-ip", cancellationToken); + var unblockIpQueueUrl = unblockIpQueue.QueueUrl; + + while(!cancellationToken.IsCancellationRequested) + { + var blockMessageResponse = await _client.ReceiveMessageAsync(new ReceiveMessageRequest + { + QueueUrl = blockIpQueueUrl, + MaxNumberOfMessages = 10, + WaitTimeSeconds = 15 + }, cancellationToken); + if(blockMessageResponse.Messages.Any()) + { + foreach(var message in blockMessageResponse.Messages) + { + try + { + await BlockIpAsync(message.Body, cancellationToken); + } + catch(Exception e) + { + _logger.LogError(e, "Failed to block IP."); + } + await _client.DeleteMessageAsync(blockIpQueueUrl, message.ReceiptHandle, cancellationToken); + } + } + + var unblockMessageResponse = await _client.ReceiveMessageAsync(new ReceiveMessageRequest + { + QueueUrl = unblockIpQueueUrl, + MaxNumberOfMessages = 10, + WaitTimeSeconds = 15 + }, cancellationToken); + if(unblockMessageResponse.Messages.Any()) + { + foreach(var message in unblockMessageResponse.Messages) + { + try + { + await UnblockIpAsync(message.Body, cancellationToken); + } + catch(Exception e) + { + _logger.LogError(e, "Failed to unblock IP."); + } + await _client.DeleteMessageAsync(unblockIpQueueUrl, message.ReceiptHandle, cancellationToken); + } + } + + await Task.Delay(TimeSpan.FromSeconds(15)); + } + } + } +} diff --git a/src/Admin/HostedServices/AzureQueueBlockIpHostedService.cs b/src/Admin/HostedServices/AzureQueueBlockIpHostedService.cs new file mode 100644 index 000000000..c0ba0f0d6 --- /dev/null +++ b/src/Admin/HostedServices/AzureQueueBlockIpHostedService.cs @@ -0,0 +1,74 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Bit.Core; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Queue; + +namespace Bit.Admin.HostedServices +{ + public class AzureQueueBlockIpHostedService : BlockIpHostedService + { + private CloudQueue _blockQueue; + private CloudQueue _unblockQueue; + + public AzureQueueBlockIpHostedService( + ILogger logger, + IOptions adminSettings, + GlobalSettings globalSettings) + : base(logger, adminSettings, globalSettings) + { } + + protected override async Task ExecuteAsync(CancellationToken cancellationToken) + { + var storageAccount = CloudStorageAccount.Parse(_globalSettings.Storage.ConnectionString); + var queueClient = storageAccount.CreateCloudQueueClient(); + _blockQueue = queueClient.GetQueueReference("blockip"); + _unblockQueue = queueClient.GetQueueReference("unblockip"); + + while(!cancellationToken.IsCancellationRequested) + { + var blockMessages = await _blockQueue.GetMessagesAsync(32, TimeSpan.FromSeconds(15), + null, null, cancellationToken); + if(blockMessages.Any()) + { + foreach(var message in blockMessages) + { + try + { + await BlockIpAsync(message.AsString, cancellationToken); + } + catch(Exception e) + { + _logger.LogError(e, "Failed to block IP."); + } + await _blockQueue.DeleteMessageAsync(message); + } + } + + var unblockMessages = await _unblockQueue.GetMessagesAsync(32, TimeSpan.FromSeconds(15), + null, null, cancellationToken); + if(unblockMessages.Any()) + { + foreach(var message in unblockMessages) + { + try + { + await UnblockIpAsync(message.AsString, cancellationToken); + } + catch(Exception e) + { + _logger.LogError(e, "Failed to unblock IP."); + } + await _unblockQueue.DeleteMessageAsync(message); + } + } + + await Task.Delay(TimeSpan.FromSeconds(15)); + } + } + } +} diff --git a/src/Admin/HostedServices/BlockIpHostedService.cs b/src/Admin/HostedServices/BlockIpHostedService.cs index 93a73d0b9..6f5272bd2 100644 --- a/src/Admin/HostedServices/BlockIpHostedService.cs +++ b/src/Admin/HostedServices/BlockIpHostedService.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Net.Http; using System.Text; using System.Threading; @@ -9,22 +8,18 @@ using Bit.Core; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Queue; using Newtonsoft.Json; namespace Bit.Admin.HostedServices { - public class BlockIpHostedService : IHostedService, IDisposable + public abstract class BlockIpHostedService : IHostedService, IDisposable { - private readonly ILogger _logger; - private readonly GlobalSettings _globalSettings; - public readonly AdminSettings _adminSettings; + protected readonly ILogger _logger; + protected readonly GlobalSettings _globalSettings; + private readonly AdminSettings _adminSettings; private Task _executingTask; private CancellationTokenSource _cts; - private CloudQueue _blockQueue; - private CloudQueue _unblockQueue; private HttpClient _httpClient = new HttpClient(); public BlockIpHostedService( @@ -55,59 +50,12 @@ namespace Bit.Admin.HostedServices cancellationToken.ThrowIfCancellationRequested(); } - public void Dispose() + public virtual void Dispose() { } - private async Task ExecuteAsync(CancellationToken cancellationToken) - { - var storageAccount = CloudStorageAccount.Parse(_globalSettings.Storage.ConnectionString); - var queueClient = storageAccount.CreateCloudQueueClient(); - _blockQueue = queueClient.GetQueueReference("blockip"); - _unblockQueue = queueClient.GetQueueReference("unblockip"); + protected abstract Task ExecuteAsync(CancellationToken cancellationToken); - while(!cancellationToken.IsCancellationRequested) - { - var blockMessages = await _blockQueue.GetMessagesAsync(32, TimeSpan.FromSeconds(15), - null, null, cancellationToken); - if(blockMessages.Any()) - { - foreach(var message in blockMessages) - { - try - { - await BlockIpAsync(message.AsString); - } - catch(Exception e) - { - _logger.LogError(e, "Failed to block IP."); - } - await _blockQueue.DeleteMessageAsync(message); - } - } - - var unblockMessages = await _unblockQueue.GetMessagesAsync(32, TimeSpan.FromSeconds(15), - null, null, cancellationToken); - if(unblockMessages.Any()) - { - foreach(var message in unblockMessages) - { - try - { - await UnblockIpAsync(message.AsString); - } - catch(Exception e) - { - _logger.LogError(e, "Failed to unblock IP."); - } - await _unblockQueue.DeleteMessageAsync(message); - } - } - - await Task.Delay(TimeSpan.FromSeconds(15)); - } - } - - private async Task BlockIpAsync(string message) + protected async Task BlockIpAsync(string message, CancellationToken cancellationToken) { var request = new HttpRequestMessage(); request.Headers.Accept.Clear(); @@ -129,7 +77,7 @@ namespace Bit.Admin.HostedServices }); request.Content = new StringContent(bodyContent, Encoding.UTF8, "application/json"); - var response = await _httpClient.SendAsync(request); + var response = await _httpClient.SendAsync(request, cancellationToken); if(!response.IsSuccessStatusCode) { return; @@ -145,7 +93,7 @@ namespace Bit.Admin.HostedServices // TODO: Send `accessRuleResponse.Result?.Id` message to unblock queue } - private async Task UnblockIpAsync(string message) + protected async Task UnblockIpAsync(string message, CancellationToken cancellationToken) { if(string.IsNullOrWhiteSpace(message)) { @@ -164,7 +112,7 @@ namespace Bit.Admin.HostedServices $"client/v4/zones/{_adminSettings.Cloudflare.ZoneId}/firewall/access_rules/rules?" + $"configuration_target=ip&configuration_value={message}"); - var response = await _httpClient.SendAsync(request); + var response = await _httpClient.SendAsync(request, cancellationToken); if(!response.IsSuccessStatusCode) { return; @@ -179,17 +127,17 @@ namespace Bit.Admin.HostedServices foreach(var rule in listResponse.Result) { - await DeleteAccessRuleAsync(rule.Id); + await DeleteAccessRuleAsync(rule.Id, cancellationToken); } } else { // Rule Id messages - await DeleteAccessRuleAsync(message); + await DeleteAccessRuleAsync(message, cancellationToken); } } - private async Task DeleteAccessRuleAsync(string ruleId) + protected async Task DeleteAccessRuleAsync(string ruleId, CancellationToken cancellationToken) { var request = new HttpRequestMessage(); request.Headers.Accept.Clear(); @@ -198,7 +146,7 @@ namespace Bit.Admin.HostedServices request.Method = HttpMethod.Delete; request.RequestUri = new Uri("https://api.cloudflare.com/" + $"client/v4/zones/{_adminSettings.Cloudflare.ZoneId}/firewall/access_rules/rules/{ruleId}"); - await _httpClient.SendAsync(request); + await _httpClient.SendAsync(request, cancellationToken); } public class ListResponse diff --git a/src/Admin/Startup.cs b/src/Admin/Startup.cs index 6ad519531..9961ed155 100644 --- a/src/Admin/Startup.cs +++ b/src/Admin/Startup.cs @@ -77,7 +77,14 @@ namespace Bit.Admin services.AddHostedService(); if(!globalSettings.SelfHosted) { - services.AddHostedService(); + if(CoreHelpers.SettingHasValue(globalSettings.Storage.ConnectionString)) + { + services.AddHostedService(); + } + else if(CoreHelpers.SettingHasValue(globalSettings.Amazon?.AccessKeySecret)) + { + services.AddHostedService(); + } } } diff --git a/src/Core/Core.csproj b/src/Core/Core.csproj index 6df6a66ea..2b90b747b 100644 --- a/src/Core/Core.csproj +++ b/src/Core/Core.csproj @@ -22,6 +22,7 @@ + diff --git a/src/Core/Services/Implementations/AmazonSqsBlockIpService.cs b/src/Core/Services/Implementations/AmazonSqsBlockIpService.cs new file mode 100644 index 000000000..9944f05f4 --- /dev/null +++ b/src/Core/Services/Implementations/AmazonSqsBlockIpService.cs @@ -0,0 +1,72 @@ +using System.Threading.Tasks; +using System; +using Amazon.SQS; +using Amazon; + +namespace Bit.Core.Services +{ + public class AmazonSqsBlockIpService : IBlockIpService, IDisposable + { + private readonly AmazonSQSClient _client; + private string _blockIpQueueUrl; + private string _unblockIpQueueUrl; + private bool _didInit = false; + private Tuple _lastBlock; + + public AmazonSqsBlockIpService( + GlobalSettings globalSettings) + { + if(string.IsNullOrWhiteSpace(globalSettings.Amazon?.AccessKeyId)) + { + throw new ArgumentNullException(nameof(globalSettings.Amazon.AccessKeyId)); + } + if(string.IsNullOrWhiteSpace(globalSettings.Amazon?.AccessKeySecret)) + { + throw new ArgumentNullException(nameof(globalSettings.Amazon.AccessKeySecret)); + } + if(string.IsNullOrWhiteSpace(globalSettings.Amazon?.Region)) + { + throw new ArgumentNullException(nameof(globalSettings.Amazon.Region)); + } + _client = new AmazonSQSClient(globalSettings.Amazon.AccessKeyId, + globalSettings.Amazon.AccessKeySecret, RegionEndpoint.GetBySystemName(globalSettings.Amazon.Region)); + } + + public void Dispose() + { + _client?.Dispose(); + } + + public async Task BlockIpAsync(string ipAddress, bool permanentBlock) + { + var now = DateTime.UtcNow; + if(_lastBlock != null && _lastBlock.Item1 == ipAddress && _lastBlock.Item2 == permanentBlock && + (now - _lastBlock.Item3) < TimeSpan.FromMinutes(1)) + { + // Already blocked this IP recently. + return; + } + + _lastBlock = new Tuple(ipAddress, permanentBlock, now); + await _client.SendMessageAsync(_blockIpQueueUrl, ipAddress); + if(!permanentBlock) + { + await _client.SendMessageAsync(_unblockIpQueueUrl, ipAddress); + } + } + + private async Task InitAsync() + { + if(_didInit) + { + return; + } + + var blockIpQueue = await _client.GetQueueUrlAsync("block-ip"); + _blockIpQueueUrl = blockIpQueue.QueueUrl; + var unblockIpQueue = await _client.GetQueueUrlAsync("unblock-ip"); + _unblockIpQueueUrl = unblockIpQueue.QueueUrl; + _didInit = true; + } + } +} diff --git a/src/Core/Utilities/ServiceCollectionExtensions.cs b/src/Core/Utilities/ServiceCollectionExtensions.cs index c9374451d..11effb798 100644 --- a/src/Core/Utilities/ServiceCollectionExtensions.cs +++ b/src/Core/Utilities/ServiceCollectionExtensions.cs @@ -121,6 +121,10 @@ namespace Bit.Core.Utilities { services.AddSingleton(); } + else if(!globalSettings.SelfHosted && CoreHelpers.SettingHasValue(globalSettings.Amazon?.AccessKeySecret)) + { + services.AddSingleton(); + } else { services.AddSingleton();