Upstream/bitwarden-server
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2024-11-25 12:45:18 +01:00
Code Issues Projects Releases Wiki Activity

Encode into b64 to avoid illegal xml encoding when sending to Azure

Browse Source
This commit is contained in:
Matt Gibson 2021-07-01 19:09:45 -04:00
parent 86a12efa76
commit d50de941da
3 changed files with 43 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/Admin/HostedServices/AzureQueueMailHostedService.cs
View File

@ -12,6 +12,7 @@ using Azure.Storage.Queues.Models;
using System.Linq;
using System.Collections.Generic;
using Newtonsoft.Json.Linq;
using Bit.Core.Utilities;
namespace Bit.Admin.HostedServices
{
@ -67,9 +68,20 @@ namespace Bit.Admin.HostedServices
foreach (var message in mailMessages)
{
// Jul 1 2021: Catch needed for now until all messages are guaranteed to be B64 strings
string messageText;
try
{
var token = JToken.Parse(message.MessageText);
messageText = CoreHelpers.Base64DecodeString(message.MessageText);
}
catch (FormatException)
{
messageText = message.MessageText;
}
try
{
var token = JToken.Parse(messageText);
if (token is JArray)
{
foreach (var mailQueueMessage in token.ToObject<List<MailQueueMessage>>())

30
src/Core/Services/Implementations/AzureQueueService.cs
View File

@ -1,9 +1,10 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations.Schema;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Azure.Storage.Queues;
using IdentityServer4.Extensions;
using Bit.Core.Utilities;
using Microsoft.EntityFrameworkCore.Internal;
using Newtonsoft.Json;
@ -23,7 +24,7 @@ namespace Bit.Core.Services
public async Task CreateAsync(T message)
{
var json = JsonConvert.SerializeObject(message, _jsonSettings);
await _queueClient.SendMessageAsync(json);
await _queueClient.SendMessageAsync(CoreHelpers.Base64EncodeString(json));
}
public async Task CreateManyAsync(IEnumerable<T> messages)
@ -39,34 +40,35 @@ namespace Bit.Core.Services
return;
}
foreach (var json in SerializeMany(messages, _jsonSettings))
foreach (var b64Json in SerializeManyToB64(messages, _jsonSettings))
{
await _queueClient.SendMessageAsync(json);
await _queueClient.SendMessageAsync(b64Json);
}
}
protected IEnumerable<string> SerializeMany(IEnumerable<T> messages, JsonSerializerSettings jsonSettings)
protected IEnumerable<string> SerializeManyToB64(IEnumerable<T> messages, JsonSerializerSettings jsonSettings)
{
var messagesLists = new List<List<T>> { new List<T>() };
var strings = new List<string>();
var ListMessageLength = 2; // to account for json array brackets "[]"
var ListMessageByteLength = 2; // to account for json array brackets "[]"
foreach (var (message, jsonEvent) in messages.Select(e => (e, JsonConvert.SerializeObject(e, jsonSettings))))
{
var messageLength = jsonEvent.Length + 1; // To account for json array comma
if (ListMessageLength + messageLength > _queueClient.MessageMaxBytes)
var messageByteLength = ByteLength(jsonEvent) + 1; // To account for json array comma
if (B64EncodedLength(ListMessageByteLength + messageByteLength) > _queueClient.MessageMaxBytes)
{
messagesLists.Add(new List<T> { message });
ListMessageLength = 2 + messageLength;
ListMessageByteLength = 2 + messageByteLength;
}
else
{
messagesLists.Last().Add(message);
ListMessageLength += messageLength;
ListMessageByteLength += messageByteLength;
}
}
return messagesLists.Select(l => JsonConvert.SerializeObject(l, jsonSettings));
return messagesLists.Select(l => CoreHelpers.Base64EncodeString(JsonConvert.SerializeObject(l, jsonSettings)));
}
private int ByteLength(string s) => Encoding.UTF8.GetByteCount(s);
private int B64EncodedLength(int byteLength) => 4 * (int)Math.Ceiling((double)byteLength / 3);
}
}

16
src/EventsProcessor/AzureQueueHostedService.cs
View File

@ -4,6 +4,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Bit.Core;
using Bit.Core.Utilities;
using Bit.Core.Models.Data;
using Bit.Core.Services;
using Microsoft.Extensions.Configuration;
@ -96,13 +97,24 @@ namespace Bit.EventsProcessor
_logger.LogWarning("Done processing.");
}
public async Task ProcessQueueMessageAsync(string message, CancellationToken cancellationToken)
public async Task ProcessQueueMessageAsync(string messageB64, CancellationToken cancellationToken)
{
if (_eventWriteService == null || message == null || message.Length == 0)
if (_eventWriteService == null || messageB64 == null || messageB64.Length == 0)
{
return;
}
// Jul 1 2021: Catch needed for now until all messages are guaranteed to be B64 strings
string message;
try
{
message = CoreHelpers.Base64DecodeString(messageB64);
}
catch (FormatException)
{
message = messageB64;
}
try
{
_logger.LogInformation("Processing message.");