Upstream/bitwarden-server
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2024-11-22 12:15:36 +01:00
Code Issues Projects Releases Wiki Activity

Fix queue message encoding for Azure (UTF-16 in XML) (#1439)

Browse Source 
* Revert "Encode into b64 to avoid illegal xml encoding when sending to Azure (#1425)"

This reverts commit 2c9a5bb4ab.

* Azure queue to use base64 encoding universally

* Ensure byte size calc is using encoded byte count

* Remove message text extension from blockIP svc

* Remove unused using on blockIp hosted service
This commit is contained in:
Chad Scharf 2021-07-07 10:49:59 -04:00 committed by GitHub
parent 908e1504af
commit 898c7baf89
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 78 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/Admin/HostedServices/AzureQueueMailHostedService.cs
View File

@ -18,7 +18,6 @@ namespace Bit.Admin.HostedServices
{
public class AzureQueueMailHostedService : IHostedService
{
private readonly JsonSerializer _jsonSerializer;
private readonly ILogger<AzureQueueMailHostedService> _logger;
private readonly GlobalSettings _globalSettings;
private readonly IMailService _mailService;
@ -35,11 +34,6 @@ namespace Bit.Admin.HostedServices
_logger = logger;
_mailService = mailService;
_globalSettings = globalSettings;
_jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings
{
Converters = new[] { new EncodedStringConverter() },
});
}
public Task StartAsync(CancellationToken cancellationToken)
@ -76,10 +70,10 @@ namespace Bit.Admin.HostedServices
{
try
{
var token = JToken.Parse(message.MessageText);
var token = JToken.Parse(message.DecodeMessageText());
if (token is JArray)
{
foreach (var mailQueueMessage in token.ToObject<List<MailQueueMessage>>(_jsonSerializer))
foreach (var mailQueueMessage in token.ToObject<List<MailQueueMessage>>())
{
await _mailService.SendEnqueuedMailMessageAsync(mailQueueMessage);
}

76
src/Core/Services/Implementations/AzureQueueService.cs
View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Azure.Storage.Queues;
using Bit.Core.Utilities;
@ -17,13 +18,14 @@ namespace Bit.Core.Services
{
_queueClient = queueClient;
_jsonSettings = jsonSettings;
if (!_jsonSettings.Converters.Any(c => c.GetType() == typeof(EncodedStringConverter)))
{
_jsonSettings.Converters.Add(new EncodedStringConverter());
}
}
public async Task CreateAsync(T message) => await CreateManyAsync(new[] { message });
public async Task CreateAsync(T message)
{
var json = JsonConvert.SerializeObject(message, _jsonSettings);
var base64 = CoreHelpers.Base64EncodeString(json);
await _queueClient.SendMessageAsync(base64);
}
public async Task CreateManyAsync(IEnumerable<T> messages)
{
@ -32,36 +34,62 @@ namespace Bit.Core.Services
return;
}
foreach (var json in SerializeMany(messages))
if (!messages.Skip(1).Any())
{
await CreateAsync(messages.First());
return;
}
foreach (var json in SerializeMany(messages, _jsonSettings))
{
await _queueClient.SendMessageAsync(json);
}
}
private IEnumerable<string> SerializeMany(IEnumerable<T> messages)
protected IEnumerable<string> SerializeMany(IEnumerable<T> messages, JsonSerializerSettings jsonSettings)
{
string SerializeMessage(T message) => JsonConvert.SerializeObject(message, _jsonSettings);
// Calculate Base-64 encoded text with padding
int getBase64Size(int byteCount) => ((4 * byteCount / 3) + 3) & ~3;
var messagesLists = new List<List<T>> { new List<T>() };
var strings = new List<string>();
var ListMessageLength = 2; // to account for json array brackets "[]"
foreach (var (message, jsonEvent) in messages.Select(m => (m, SerializeMessage(m))))
var messagesList = new List<string>();
var messagesListSize = 0;
int calculateByteSize(int totalSize, int toAdd) =>
// Calculate the total length this would be w/ "[]" and commas
getBase64Size(totalSize + toAdd + messagesList.Count + 2);
// Format the final array string, i.e. [{...},{...}]
string getArrayString()
{
var messageLength = jsonEvent.Length + 1; // To account for json array comma
if (ListMessageLength + messageLength > _queueClient.MessageMaxBytes)
if (messagesList.Count == 1)
{
messagesLists.Add(new List<T> { message });
ListMessageLength = 2 + messageLength;
}
else
{
messagesLists.Last().Add(message);
ListMessageLength += messageLength;
return CoreHelpers.Base64EncodeString(messagesList[0]);
}
return CoreHelpers.Base64EncodeString(
string.Concat("[", string.Join(',', messagesList), "]"));
}
var serializedMessages = messages.Select(message =>
JsonConvert.SerializeObject(message, jsonSettings));
foreach (var message in serializedMessages)
{
var messageSize = Encoding.UTF8.GetByteCount(message);
if (calculateByteSize(messagesListSize, messageSize) > _queueClient.MessageMaxBytes)
{
yield return getArrayString();
messagesListSize = 0;
messagesList.Clear();
}
messagesList.Add(message);
messagesListSize += messageSize;
}
if (messagesList.Any())
{
yield return getArrayString();
}
return messagesLists.Select(l => JsonConvert.SerializeObject(l, _jsonSettings));
}
}
}

20
src/Core/Utilities/CoreHelpers.cs
View File

@ -24,6 +24,9 @@ using Bit.Core.Models.Table;
using IdentityModel;
using System.Text.Json;
using Bit.Core.Enums.Provider;
using Azure.Storage.Queues;
using Azure.Storage.Queues.Models;
using System.Threading;
namespace Bit.Core.Utilities
{
@ -901,5 +904,22 @@ namespace Bit.Core.Utilities
list.Add(item);
return list;
}
public static string DecodeMessageText(this QueueMessage message)
{
var text = message?.MessageText;
if (string.IsNullOrWhiteSpace(text))
{
return text;
}
try
{
return Base64DecodeString(text);
}
catch
{
return text;
}
}
}
}

35
src/Core/Utilities/EncodedStringConverter.cs
View File

@ -1,35 +0,0 @@
using System;
using Newtonsoft.Json;
namespace Bit.Core.Utilities
{
public class EncodedStringConverter : JsonConverter
{
public override bool CanConvert(Type objectType) => objectType == typeof(string);
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null)
{
return existingValue;
}
var value = reader.Value as string;
return System.Net.WebUtility.HtmlDecode(value);
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
if (value == null)
{
if (serializer.NullValueHandling == NullValueHandling.Include)
{
writer.WriteNull();
}
return;
}
writer.WriteValue(System.Net.WebUtility.HtmlEncode((string)value));
}
}
}

10
src/EventsProcessor/AzureQueueHostedService.cs
View File

@ -18,7 +18,6 @@ namespace Bit.EventsProcessor
{
public class AzureQueueHostedService : IHostedService, IDisposable
{
private readonly JsonSerializer _jsonSerializer;
private readonly ILogger<AzureQueueHostedService> _logger;
private readonly IConfiguration _configuration;
@ -33,11 +32,6 @@ namespace Bit.EventsProcessor
{
_logger = logger;
_configuration = configuration;
_jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings
{
Converters = new[] { new EncodedStringConverter() },
});
}
public Task StartAsync(CancellationToken cancellationToken)
@ -84,7 +78,7 @@ namespace Bit.EventsProcessor
{
foreach (var message in messages.Value)
{
await ProcessQueueMessageAsync(message.MessageText, cancellationToken);
await ProcessQueueMessageAsync(message.DecodeMessageText(), cancellationToken);
await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt);
}
}
@ -118,7 +112,7 @@ namespace Bit.EventsProcessor
var token = JToken.Parse(message);
if (token is JArray)
{
var indexedEntities = token.ToObject<List<EventMessage>>(_jsonSerializer)
var indexedEntities = token.ToObject<List<EventMessage>>()
.SelectMany(e => EventTableEntity.IndexEvent(e));
events.AddRange(indexedEntities);
}

3
src/Notifications/AzureQueueHostedService.cs
View File

@ -7,6 +7,7 @@ using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Azure.Storage.Queues;
using Bit.Core.Utilities;
namespace Bit.Notifications
{
@ -67,7 +68,7 @@ namespace Bit.Notifications
try
{
await HubHelpers.SendNotificationToHubAsync(
message.MessageText, _hubContext, cancellationToken);
message.DecodeMessageText(), _hubContext, cancellationToken);
await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt);
}
catch (Exception e)