Encode into b64 to avoid illegal xml encoding when sending to Azure (#1425)

* Encode into b64 to avoid illegal xml encoding when sending to Azure

* Revert "Encode into b64 to avoid illegal xml encoding when sending to Azure"

This reverts commit d50de941da.

* HtmlEncode strings if they use multi-byte characters

* Add serializer to event processor

* Rename to used class

* Formatting

* PR feedback

src/Admin/HostedServices/AzureQueueMailHostedService.cs

@@ -12,11 +12,13 @@ using Azure.Storage.Queues.Models;
 using System.Linq;
 using System.Collections.Generic;
 using Newtonsoft.Json.Linq;
+using Bit.Core.Utilities;
 
 namespace Bit.Admin.HostedServices
 {
     public class AzureQueueMailHostedService : IHostedService
     {
+        private readonly JsonSerializer _jsonSerializer;
         private readonly ILogger<AzureQueueMailHostedService> _logger;
         private readonly GlobalSettings _globalSettings;
         private readonly IMailService _mailService;
@@ -33,6 +35,11 @@ namespace Bit.Admin.HostedServices
             _logger = logger;
             _mailService = mailService;
             _globalSettings = globalSettings;
+
+            _jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings
+            {
+                Converters = new[] { new EncodedStringConverter() },
+            });
         }
 
         public Task StartAsync(CancellationToken cancellationToken)
@@ -72,7 +79,7 @@ namespace Bit.Admin.HostedServices
                         var token = JToken.Parse(message.MessageText);
                         if (token is JArray)
                         {
-                            foreach (var mailQueueMessage in token.ToObject<List<MailQueueMessage>>())
+                            foreach (var mailQueueMessage in token.ToObject<List<MailQueueMessage>>(_jsonSerializer))
                             {
                                 await _mailService.SendEnqueuedMailMessageAsync(mailQueueMessage);
                             }

src/Core/Services/Implementations/AzureQueueService.cs

@@ -1,9 +1,8 @@
 using System.Collections.Generic;
-using System.ComponentModel.DataAnnotations.Schema;
 using System.Linq;
 using System.Threading.Tasks;
 using Azure.Storage.Queues;
-using IdentityServer4.Extensions;
+using Bit.Core.Utilities;
 using Microsoft.EntityFrameworkCore.Internal;
 using Newtonsoft.Json;
 
@@ -18,13 +17,13 @@ namespace Bit.Core.Services
         {
             _queueClient = queueClient;
             _jsonSettings = jsonSettings;
+            if (!_jsonSettings.Converters.Any(c => c.GetType() == typeof(EncodedStringConverter)))
+            {
+                _jsonSettings.Converters.Add(new EncodedStringConverter());
+            }
         }
 
-        public async Task CreateAsync(T message)
+        public async Task CreateAsync(T message) => await CreateManyAsync(new[] { message });
-        {
-            var json = JsonConvert.SerializeObject(message, _jsonSettings);
-            await _queueClient.SendMessageAsync(json);
-        }
 
         public async Task CreateManyAsync(IEnumerable<T> messages)
         {
@@ -33,25 +32,21 @@ namespace Bit.Core.Services
                 return;
             }
 
-            if (!messages.Skip(1).Any())
+            foreach (var json in SerializeMany(messages))
-            {
-                await CreateAsync(messages.First());
-                return;
-            }
-
-            foreach (var json in SerializeMany(messages, _jsonSettings))
             {
                 await _queueClient.SendMessageAsync(json);
             }
         }
 
 
-        protected IEnumerable<string> SerializeMany(IEnumerable<T> messages, JsonSerializerSettings jsonSettings)
+        private IEnumerable<string> SerializeMany(IEnumerable<T> messages)
         {
+            string SerializeMessage(T message) => JsonConvert.SerializeObject(message, _jsonSettings);
+
             var messagesLists = new List<List<T>> { new List<T>() };
             var strings = new List<string>();
             var ListMessageLength = 2; // to account for json array brackets "[]"
-            foreach (var (message, jsonEvent) in messages.Select(e => (e, JsonConvert.SerializeObject(e, jsonSettings))))
+            foreach (var (message, jsonEvent) in messages.Select(m => (m, SerializeMessage(m))))
             {
 
                 var messageLength = jsonEvent.Length + 1; // To account for json array comma
@@ -66,7 +61,7 @@ namespace Bit.Core.Services
                     ListMessageLength += messageLength;
                 }
             }
-            return messagesLists.Select(l => JsonConvert.SerializeObject(l, jsonSettings));
+            return messagesLists.Select(l => JsonConvert.SerializeObject(l, _jsonSettings));
         }
     }
 }

src/Core/Utilities/EncodedStringConverter.cs

@@ -0,0 +1,35 @@
+using System;
+using Newtonsoft.Json;
+
+namespace Bit.Core.Utilities
+{
+    public class EncodedStringConverter : JsonConverter
+    {
+        public override bool CanConvert(Type objectType) => objectType == typeof(string);
+
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            if (reader.TokenType == JsonToken.Null)
+            {
+                return existingValue;
+            }
+
+            var value = reader.Value as string;
+            return System.Net.WebUtility.HtmlDecode(value);
+        }
+
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            if (value == null)
+            {
+                if (serializer.NullValueHandling == NullValueHandling.Include)
+                {
+                    writer.WriteNull();
+                }
+                return;
+            }
+
+            writer.WriteValue(System.Net.WebUtility.HtmlEncode((string)value));
+        }
+    }
+}

src/EventsProcessor/AzureQueueHostedService.cs

@@ -12,11 +12,13 @@ using Microsoft.Extensions.Logging;
 using Azure.Storage.Queues;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
+using Bit.Core.Utilities;
 
 namespace Bit.EventsProcessor
 {
     public class AzureQueueHostedService : IHostedService, IDisposable
     {
+        private readonly JsonSerializer _jsonSerializer;
         private readonly ILogger<AzureQueueHostedService> _logger;
         private readonly IConfiguration _configuration;
 
@@ -31,6 +33,11 @@ namespace Bit.EventsProcessor
         {
             _logger = logger;
             _configuration = configuration;
+
+            _jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings
+            {
+                Converters = new[] { new EncodedStringConverter() },
+            });
         }
 
         public Task StartAsync(CancellationToken cancellationToken)
@@ -111,7 +118,7 @@ namespace Bit.EventsProcessor
                 var token = JToken.Parse(message);
                 if (token is JArray)
                 {
-                    var indexedEntities = token.ToObject<List<EventMessage>>()
+                    var indexedEntities = token.ToObject<List<EventMessage>>(_jsonSerializer)
                         .SelectMany(e => EventTableEntity.IndexEvent(e));
                     events.AddRange(indexedEntities);
                 }