Fix queue message encoding for Azure (UTF-16 in XML) (#1439)

* Revert "Encode into b64 to avoid illegal xml encoding when sending to Azure (#1425)"

This reverts commit 2c9a5bb4ab.

* Azure queue to use base64 encoding universally

* Ensure byte size calc is using encoded byte count

* Remove message text extension from blockIP svc

* Remove unused using on blockIp hosted service

src/Admin/HostedServices/AzureQueueMailHostedService.cs

@@ -18,7 +18,6 @@ namespace Bit.Admin.HostedServices
 {
     public class AzureQueueMailHostedService : IHostedService
     {
-        private readonly JsonSerializer _jsonSerializer;
         private readonly ILogger<AzureQueueMailHostedService> _logger;
         private readonly GlobalSettings _globalSettings;
         private readonly IMailService _mailService;
@@ -35,11 +34,6 @@ namespace Bit.Admin.HostedServices
             _logger = logger;
             _mailService = mailService;
             _globalSettings = globalSettings;
-
-            _jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings
-            {
-                Converters = new[] { new EncodedStringConverter() },
-            });
         }
 
         public Task StartAsync(CancellationToken cancellationToken)
@@ -76,10 +70,10 @@ namespace Bit.Admin.HostedServices
                 {
                     try
                     {
-                        var token = JToken.Parse(message.MessageText);
+                        var token = JToken.Parse(message.DecodeMessageText());
                         if (token is JArray)
                         {
-                            foreach (var mailQueueMessage in token.ToObject<List<MailQueueMessage>>(_jsonSerializer))
+                            foreach (var mailQueueMessage in token.ToObject<List<MailQueueMessage>>())
                             {
                                 await _mailService.SendEnqueuedMailMessageAsync(mailQueueMessage);
                             }

src/Core/Services/Implementations/AzureQueueService.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using System.Threading.Tasks;
 using Azure.Storage.Queues;
 using Bit.Core.Utilities;
@@ -17,13 +18,14 @@ namespace Bit.Core.Services
         {
             _queueClient = queueClient;
             _jsonSettings = jsonSettings;
-            if (!_jsonSettings.Converters.Any(c => c.GetType() == typeof(EncodedStringConverter)))
-            {
-                _jsonSettings.Converters.Add(new EncodedStringConverter());
-            }
         }
 
-        public async Task CreateAsync(T message) => await CreateManyAsync(new[] { message });
+        public async Task CreateAsync(T message)
+        {
+            var json = JsonConvert.SerializeObject(message, _jsonSettings);
+            var base64 = CoreHelpers.Base64EncodeString(json);
+            await _queueClient.SendMessageAsync(base64);
+        }
 
         public async Task CreateManyAsync(IEnumerable<T> messages)
         {
@@ -32,36 +34,62 @@ namespace Bit.Core.Services
                 return;
             }
 
-            foreach (var json in SerializeMany(messages))
+            if (!messages.Skip(1).Any())
+            {
+                await CreateAsync(messages.First());
+                return;
+            }
+
+            foreach (var json in SerializeMany(messages, _jsonSettings))
             {
                 await _queueClient.SendMessageAsync(json);
             }
         }
 
+        protected IEnumerable<string> SerializeMany(IEnumerable<T> messages, JsonSerializerSettings jsonSettings)
+        {
+            // Calculate Base-64 encoded text with padding
+            int getBase64Size(int byteCount) => ((4 * byteCount / 3) + 3) & ~3;
 
-        private IEnumerable<string> SerializeMany(IEnumerable<T> messages)
+            var messagesList = new List<string>();
-        {
+            var messagesListSize = 0;
-            string SerializeMessage(T message) => JsonConvert.SerializeObject(message, _jsonSettings);
             
-            var messagesLists = new List<List<T>> { new List<T>() };
+            int calculateByteSize(int totalSize, int toAdd) =>
-            var strings = new List<string>();
+                // Calculate the total length this would be w/ "[]" and commas
-            var ListMessageLength = 2; // to account for json array brackets "[]"
+                getBase64Size(totalSize + toAdd + messagesList.Count + 2);
-            foreach (var (message, jsonEvent) in messages.Select(m => (m, SerializeMessage(m))))
-            {
 
-                var messageLength = jsonEvent.Length + 1; // To account for json array comma
+            // Format the final array string, i.e. [{...},{...}]
-                if (ListMessageLength + messageLength > _queueClient.MessageMaxBytes)
+            string getArrayString()
             {
-                    messagesLists.Add(new List<T> { message });
+                if (messagesList.Count == 1)
-                    ListMessageLength = 2 + messageLength;
-                }
-                else
                 {
-                    messagesLists.Last().Add(message);
+                    return CoreHelpers.Base64EncodeString(messagesList[0]);
-                    ListMessageLength += messageLength;
                 }
+                return CoreHelpers.Base64EncodeString(
+                    string.Concat("[", string.Join(',', messagesList), "]"));
+            }
+            
+            var serializedMessages = messages.Select(message =>
+                JsonConvert.SerializeObject(message, jsonSettings));
+
+            foreach (var message in serializedMessages)
+            {
+                var messageSize = Encoding.UTF8.GetByteCount(message);
+                if (calculateByteSize(messagesListSize, messageSize) > _queueClient.MessageMaxBytes)
+                {
+                    yield return getArrayString();
+                    messagesListSize = 0;
+                    messagesList.Clear();
+                }
+
+                messagesList.Add(message);
+                messagesListSize += messageSize;
+            }
+
+            if (messagesList.Any())
+            {
+                yield return getArrayString();
             }
-            return messagesLists.Select(l => JsonConvert.SerializeObject(l, _jsonSettings));
         }
     }
 }

src/Core/Utilities/CoreHelpers.cs

@@ -24,6 +24,9 @@ using Bit.Core.Models.Table;
 using IdentityModel;
 using System.Text.Json;
 using Bit.Core.Enums.Provider;
+using Azure.Storage.Queues;
+using Azure.Storage.Queues.Models;
+using System.Threading;
 
 namespace Bit.Core.Utilities
 {
@@ -901,5 +904,22 @@ namespace Bit.Core.Utilities
             list.Add(item);
             return list;
         }
+
+        public static string DecodeMessageText(this QueueMessage message)
+        {
+            var text = message?.MessageText;
+            if (string.IsNullOrWhiteSpace(text))
+            {
+                return text;
+            }
+            try
+            {
+                return Base64DecodeString(text);
+            }
+            catch
+            {
+                return text;
+            }
+        }
     }
 }

src/Core/Utilities/EncodedStringConverter.cs

@@ -1,35 +0,0 @@
-using System;
-using Newtonsoft.Json;
-
-namespace Bit.Core.Utilities
-{
-    public class EncodedStringConverter : JsonConverter
-    {
-        public override bool CanConvert(Type objectType) => objectType == typeof(string);
-
-        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
-        {
-            if (reader.TokenType == JsonToken.Null)
-            {
-                return existingValue;
-            }
-
-            var value = reader.Value as string;
-            return System.Net.WebUtility.HtmlDecode(value);
-        }
-
-        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
-        {
-            if (value == null)
-            {
-                if (serializer.NullValueHandling == NullValueHandling.Include)
-                {
-                    writer.WriteNull();
-                }
-                return;
-            }
-
-            writer.WriteValue(System.Net.WebUtility.HtmlEncode((string)value));
-        }
-    }
-}

src/EventsProcessor/AzureQueueHostedService.cs

@@ -18,7 +18,6 @@ namespace Bit.EventsProcessor
 {
     public class AzureQueueHostedService : IHostedService, IDisposable
     {
-        private readonly JsonSerializer _jsonSerializer;
         private readonly ILogger<AzureQueueHostedService> _logger;
         private readonly IConfiguration _configuration;
 
@@ -33,11 +32,6 @@ namespace Bit.EventsProcessor
         {
             _logger = logger;
             _configuration = configuration;
-
-            _jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings
-            {
-                Converters = new[] { new EncodedStringConverter() },
-            });
         }
 
         public Task StartAsync(CancellationToken cancellationToken)
@@ -84,7 +78,7 @@ namespace Bit.EventsProcessor
                     {
                         foreach (var message in messages.Value)
                         {
-                            await ProcessQueueMessageAsync(message.MessageText, cancellationToken);
+                            await ProcessQueueMessageAsync(message.DecodeMessageText(), cancellationToken);
                             await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt);
                         }
                     }
@@ -118,7 +112,7 @@ namespace Bit.EventsProcessor
                 var token = JToken.Parse(message);
                 if (token is JArray)
                 {
-                    var indexedEntities = token.ToObject<List<EventMessage>>(_jsonSerializer)
+                    var indexedEntities = token.ToObject<List<EventMessage>>()
                         .SelectMany(e => EventTableEntity.IndexEvent(e));
                     events.AddRange(indexedEntities);
                 }

src/Notifications/AzureQueueHostedService.cs

@@ -7,6 +7,7 @@ using Microsoft.AspNetCore.SignalR;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Azure.Storage.Queues;
+using Bit.Core.Utilities;
 
 namespace Bit.Notifications
 {
@@ -67,7 +68,7 @@ namespace Bit.Notifications
                             try
                             {
                                 await HubHelpers.SendNotificationToHubAsync(
-                                    message.MessageText, _hubContext, cancellationToken);
+                                    message.DecodeMessageText(), _hubContext, cancellationToken);
                                 await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt);
                             }
                             catch (Exception e)