From 898c7baf893f99b145871d42480db7e4d5a42956 Mon Sep 17 00:00:00 2001 From: Chad Scharf <3904944+cscharf@users.noreply.github.com> Date: Wed, 7 Jul 2021 10:49:59 -0400 Subject: [PATCH] Fix queue message encoding for Azure (UTF-16 in XML) (#1439) * Revert "Encode into b64 to avoid illegal xml encoding when sending to Azure (#1425)" This reverts commit 2c9a5bb4ab58ab68192b2fb94ff459b7326a99b1. * Azure queue to use base64 encoding universally * Ensure byte size calc is using encoded byte count * Remove message text extension from blockIP svc * Remove unused using on blockIp hosted service --- .../AzureQueueMailHostedService.cs | 10 +-- .../Implementations/AzureQueueService.cs | 76 +++++++++++++------ src/Core/Utilities/CoreHelpers.cs | 20 +++++ src/Core/Utilities/EncodedStringConverter.cs | 35 --------- .../AzureQueueHostedService.cs | 10 +-- src/Notifications/AzureQueueHostedService.cs | 3 +- 6 files changed, 78 insertions(+), 76 deletions(-) delete mode 100644 src/Core/Utilities/EncodedStringConverter.cs diff --git a/src/Admin/HostedServices/AzureQueueMailHostedService.cs b/src/Admin/HostedServices/AzureQueueMailHostedService.cs index 67e8623d7..17d4b2fc5 100644 --- a/src/Admin/HostedServices/AzureQueueMailHostedService.cs +++ b/src/Admin/HostedServices/AzureQueueMailHostedService.cs @@ -18,7 +18,6 @@ namespace Bit.Admin.HostedServices { public class AzureQueueMailHostedService : IHostedService { - private readonly JsonSerializer _jsonSerializer; private readonly ILogger _logger; private readonly GlobalSettings _globalSettings; private readonly IMailService _mailService; @@ -35,11 +34,6 @@ namespace Bit.Admin.HostedServices _logger = logger; _mailService = mailService; _globalSettings = globalSettings; - - _jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings - { - Converters = new[] { new EncodedStringConverter() }, - }); } public Task StartAsync(CancellationToken cancellationToken) @@ -76,10 +70,10 @@ namespace Bit.Admin.HostedServices { try { - var token = JToken.Parse(message.MessageText); + var token = JToken.Parse(message.DecodeMessageText()); if (token is JArray) { - foreach (var mailQueueMessage in token.ToObject>(_jsonSerializer)) + foreach (var mailQueueMessage in token.ToObject>()) { await _mailService.SendEnqueuedMailMessageAsync(mailQueueMessage); } diff --git a/src/Core/Services/Implementations/AzureQueueService.cs b/src/Core/Services/Implementations/AzureQueueService.cs index bb5d89595..a8162c5e1 100644 --- a/src/Core/Services/Implementations/AzureQueueService.cs +++ b/src/Core/Services/Implementations/AzureQueueService.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading.Tasks; using Azure.Storage.Queues; using Bit.Core.Utilities; @@ -17,13 +18,14 @@ namespace Bit.Core.Services { _queueClient = queueClient; _jsonSettings = jsonSettings; - if (!_jsonSettings.Converters.Any(c => c.GetType() == typeof(EncodedStringConverter))) - { - _jsonSettings.Converters.Add(new EncodedStringConverter()); - } } - public async Task CreateAsync(T message) => await CreateManyAsync(new[] { message }); + public async Task CreateAsync(T message) + { + var json = JsonConvert.SerializeObject(message, _jsonSettings); + var base64 = CoreHelpers.Base64EncodeString(json); + await _queueClient.SendMessageAsync(base64); + } public async Task CreateManyAsync(IEnumerable messages) { @@ -32,36 +34,62 @@ namespace Bit.Core.Services return; } - foreach (var json in SerializeMany(messages)) + if (!messages.Skip(1).Any()) + { + await CreateAsync(messages.First()); + return; + } + + foreach (var json in SerializeMany(messages, _jsonSettings)) { await _queueClient.SendMessageAsync(json); } } - - private IEnumerable SerializeMany(IEnumerable messages) + protected IEnumerable SerializeMany(IEnumerable messages, JsonSerializerSettings jsonSettings) { - string SerializeMessage(T message) => JsonConvert.SerializeObject(message, _jsonSettings); + // Calculate Base-64 encoded text with padding + int getBase64Size(int byteCount) => ((4 * byteCount / 3) + 3) & ~3; - var messagesLists = new List> { new List() }; - var strings = new List(); - var ListMessageLength = 2; // to account for json array brackets "[]" - foreach (var (message, jsonEvent) in messages.Select(m => (m, SerializeMessage(m)))) + var messagesList = new List(); + var messagesListSize = 0; + + int calculateByteSize(int totalSize, int toAdd) => + // Calculate the total length this would be w/ "[]" and commas + getBase64Size(totalSize + toAdd + messagesList.Count + 2); + + // Format the final array string, i.e. [{...},{...}] + string getArrayString() { - - var messageLength = jsonEvent.Length + 1; // To account for json array comma - if (ListMessageLength + messageLength > _queueClient.MessageMaxBytes) + if (messagesList.Count == 1) { - messagesLists.Add(new List { message }); - ListMessageLength = 2 + messageLength; - } - else - { - messagesLists.Last().Add(message); - ListMessageLength += messageLength; + return CoreHelpers.Base64EncodeString(messagesList[0]); } + return CoreHelpers.Base64EncodeString( + string.Concat("[", string.Join(',', messagesList), "]")); + } + + var serializedMessages = messages.Select(message => + JsonConvert.SerializeObject(message, jsonSettings)); + + foreach (var message in serializedMessages) + { + var messageSize = Encoding.UTF8.GetByteCount(message); + if (calculateByteSize(messagesListSize, messageSize) > _queueClient.MessageMaxBytes) + { + yield return getArrayString(); + messagesListSize = 0; + messagesList.Clear(); + } + + messagesList.Add(message); + messagesListSize += messageSize; + } + + if (messagesList.Any()) + { + yield return getArrayString(); } - return messagesLists.Select(l => JsonConvert.SerializeObject(l, _jsonSettings)); } } } diff --git a/src/Core/Utilities/CoreHelpers.cs b/src/Core/Utilities/CoreHelpers.cs index c00703702..58de340a8 100644 --- a/src/Core/Utilities/CoreHelpers.cs +++ b/src/Core/Utilities/CoreHelpers.cs @@ -24,6 +24,9 @@ using Bit.Core.Models.Table; using IdentityModel; using System.Text.Json; using Bit.Core.Enums.Provider; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using System.Threading; namespace Bit.Core.Utilities { @@ -901,5 +904,22 @@ namespace Bit.Core.Utilities list.Add(item); return list; } + + public static string DecodeMessageText(this QueueMessage message) + { + var text = message?.MessageText; + if (string.IsNullOrWhiteSpace(text)) + { + return text; + } + try + { + return Base64DecodeString(text); + } + catch + { + return text; + } + } } } diff --git a/src/Core/Utilities/EncodedStringConverter.cs b/src/Core/Utilities/EncodedStringConverter.cs deleted file mode 100644 index 5bb033d5f..000000000 --- a/src/Core/Utilities/EncodedStringConverter.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System; -using Newtonsoft.Json; - -namespace Bit.Core.Utilities -{ - public class EncodedStringConverter : JsonConverter - { - public override bool CanConvert(Type objectType) => objectType == typeof(string); - - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) - { - if (reader.TokenType == JsonToken.Null) - { - return existingValue; - } - - var value = reader.Value as string; - return System.Net.WebUtility.HtmlDecode(value); - } - - public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) - { - if (value == null) - { - if (serializer.NullValueHandling == NullValueHandling.Include) - { - writer.WriteNull(); - } - return; - } - - writer.WriteValue(System.Net.WebUtility.HtmlEncode((string)value)); - } - } -} diff --git a/src/EventsProcessor/AzureQueueHostedService.cs b/src/EventsProcessor/AzureQueueHostedService.cs index 470f646d1..a5d86d661 100644 --- a/src/EventsProcessor/AzureQueueHostedService.cs +++ b/src/EventsProcessor/AzureQueueHostedService.cs @@ -18,7 +18,6 @@ namespace Bit.EventsProcessor { public class AzureQueueHostedService : IHostedService, IDisposable { - private readonly JsonSerializer _jsonSerializer; private readonly ILogger _logger; private readonly IConfiguration _configuration; @@ -33,11 +32,6 @@ namespace Bit.EventsProcessor { _logger = logger; _configuration = configuration; - - _jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings - { - Converters = new[] { new EncodedStringConverter() }, - }); } public Task StartAsync(CancellationToken cancellationToken) @@ -84,7 +78,7 @@ namespace Bit.EventsProcessor { foreach (var message in messages.Value) { - await ProcessQueueMessageAsync(message.MessageText, cancellationToken); + await ProcessQueueMessageAsync(message.DecodeMessageText(), cancellationToken); await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt); } } @@ -118,7 +112,7 @@ namespace Bit.EventsProcessor var token = JToken.Parse(message); if (token is JArray) { - var indexedEntities = token.ToObject>(_jsonSerializer) + var indexedEntities = token.ToObject>() .SelectMany(e => EventTableEntity.IndexEvent(e)); events.AddRange(indexedEntities); } diff --git a/src/Notifications/AzureQueueHostedService.cs b/src/Notifications/AzureQueueHostedService.cs index 85a7038dd..e90203f27 100644 --- a/src/Notifications/AzureQueueHostedService.cs +++ b/src/Notifications/AzureQueueHostedService.cs @@ -7,6 +7,7 @@ using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Azure.Storage.Queues; +using Bit.Core.Utilities; namespace Bit.Notifications { @@ -67,7 +68,7 @@ namespace Bit.Notifications try { await HubHelpers.SendNotificationToHubAsync( - message.MessageText, _hubContext, cancellationToken); + message.DecodeMessageText(), _hubContext, cancellationToken); await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt); } catch (Exception e)