From d50de941da656ac7b1622da5e68a0e98169d690a Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Thu, 1 Jul 2021 19:09:45 -0400 Subject: [PATCH] Encode into b64 to avoid illegal xml encoding when sending to Azure --- .../AzureQueueMailHostedService.cs | 14 ++++++++- .../Implementations/AzureQueueService.cs | 30 ++++++++++--------- .../AzureQueueHostedService.cs | 16 ++++++++-- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/Admin/HostedServices/AzureQueueMailHostedService.cs b/src/Admin/HostedServices/AzureQueueMailHostedService.cs index 790643f3c..60b79d3ef 100644 --- a/src/Admin/HostedServices/AzureQueueMailHostedService.cs +++ b/src/Admin/HostedServices/AzureQueueMailHostedService.cs @@ -12,6 +12,7 @@ using Azure.Storage.Queues.Models; using System.Linq; using System.Collections.Generic; using Newtonsoft.Json.Linq; +using Bit.Core.Utilities; namespace Bit.Admin.HostedServices { @@ -67,9 +68,20 @@ namespace Bit.Admin.HostedServices foreach (var message in mailMessages) { + // Jul 1 2021: Catch needed for now until all messages are guaranteed to be B64 strings + string messageText; try { - var token = JToken.Parse(message.MessageText); + messageText = CoreHelpers.Base64DecodeString(message.MessageText); + } + catch (FormatException) + { + messageText = message.MessageText; + } + + try + { + var token = JToken.Parse(messageText); if (token is JArray) { foreach (var mailQueueMessage in token.ToObject>()) diff --git a/src/Core/Services/Implementations/AzureQueueService.cs b/src/Core/Services/Implementations/AzureQueueService.cs index faccc123a..3357a9a67 100644 --- a/src/Core/Services/Implementations/AzureQueueService.cs +++ b/src/Core/Services/Implementations/AzureQueueService.cs @@ -1,9 +1,10 @@ +using System; using System.Collections.Generic; -using System.ComponentModel.DataAnnotations.Schema; using System.Linq; +using System.Text; using System.Threading.Tasks; using Azure.Storage.Queues; -using IdentityServer4.Extensions; +using Bit.Core.Utilities; using Microsoft.EntityFrameworkCore.Internal; using Newtonsoft.Json; @@ -23,7 +24,7 @@ namespace Bit.Core.Services public async Task CreateAsync(T message) { var json = JsonConvert.SerializeObject(message, _jsonSettings); - await _queueClient.SendMessageAsync(json); + await _queueClient.SendMessageAsync(CoreHelpers.Base64EncodeString(json)); } public async Task CreateManyAsync(IEnumerable messages) @@ -39,34 +40,35 @@ namespace Bit.Core.Services return; } - foreach (var json in SerializeMany(messages, _jsonSettings)) + foreach (var b64Json in SerializeManyToB64(messages, _jsonSettings)) { - await _queueClient.SendMessageAsync(json); + await _queueClient.SendMessageAsync(b64Json); } } - - protected IEnumerable SerializeMany(IEnumerable messages, JsonSerializerSettings jsonSettings) + protected IEnumerable SerializeManyToB64(IEnumerable messages, JsonSerializerSettings jsonSettings) { var messagesLists = new List> { new List() }; var strings = new List(); - var ListMessageLength = 2; // to account for json array brackets "[]" + var ListMessageByteLength = 2; // to account for json array brackets "[]" foreach (var (message, jsonEvent) in messages.Select(e => (e, JsonConvert.SerializeObject(e, jsonSettings)))) { - - var messageLength = jsonEvent.Length + 1; // To account for json array comma - if (ListMessageLength + messageLength > _queueClient.MessageMaxBytes) + var messageByteLength = ByteLength(jsonEvent) + 1; // To account for json array comma + if (B64EncodedLength(ListMessageByteLength + messageByteLength) > _queueClient.MessageMaxBytes) { messagesLists.Add(new List { message }); - ListMessageLength = 2 + messageLength; + ListMessageByteLength = 2 + messageByteLength; } else { messagesLists.Last().Add(message); - ListMessageLength += messageLength; + ListMessageByteLength += messageByteLength; } } - return messagesLists.Select(l => JsonConvert.SerializeObject(l, jsonSettings)); + return messagesLists.Select(l => CoreHelpers.Base64EncodeString(JsonConvert.SerializeObject(l, jsonSettings))); } + + private int ByteLength(string s) => Encoding.UTF8.GetByteCount(s); + private int B64EncodedLength(int byteLength) => 4 * (int)Math.Ceiling((double)byteLength / 3); } } diff --git a/src/EventsProcessor/AzureQueueHostedService.cs b/src/EventsProcessor/AzureQueueHostedService.cs index 9964e67a2..aeee8bb9a 100644 --- a/src/EventsProcessor/AzureQueueHostedService.cs +++ b/src/EventsProcessor/AzureQueueHostedService.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using Bit.Core; +using Bit.Core.Utilities; using Bit.Core.Models.Data; using Bit.Core.Services; using Microsoft.Extensions.Configuration; @@ -96,13 +97,24 @@ namespace Bit.EventsProcessor _logger.LogWarning("Done processing."); } - public async Task ProcessQueueMessageAsync(string message, CancellationToken cancellationToken) + public async Task ProcessQueueMessageAsync(string messageB64, CancellationToken cancellationToken) { - if (_eventWriteService == null || message == null || message.Length == 0) + if (_eventWriteService == null || messageB64 == null || messageB64.Length == 0) { return; } + // Jul 1 2021: Catch needed for now until all messages are guaranteed to be B64 strings + string message; + try + { + message = CoreHelpers.Base64DecodeString(messageB64); + } + catch (FormatException) + { + message = messageB64; + } + try { _logger.LogInformation("Processing message.");