1
0
mirror of https://github.com/bitwarden/server.git synced 2025-01-09 19:57:37 +01:00

[PM-6762] Move to Azure.Data.Tables (#3888)

* Move to Azure.Data.Tables

* Reorder usings

* Add new package to Renovate

* Add manual serialization and deserialization due to enums

* Properly retrieve just the next page
This commit is contained in:
Matt Bishop 2024-03-28 16:36:24 -04:00 committed by GitHub
parent ffd988eeda
commit c53e5eeab3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 144 additions and 280 deletions

View File

@ -44,6 +44,7 @@
"matchPackageNames": [ "matchPackageNames": [
"AspNetCoreRateLimit", "AspNetCoreRateLimit",
"AspNetCoreRateLimit.Redis", "AspNetCoreRateLimit.Redis",
"Azure.Data.Tables",
"Azure.Extensions.AspNetCore.DataProtection.Blobs", "Azure.Extensions.AspNetCore.DataProtection.Blobs",
"Azure.Messaging.EventGrid", "Azure.Messaging.EventGrid",
"Azure.Messaging.ServiceBus", "Azure.Messaging.ServiceBus",
@ -53,7 +54,6 @@
"Fido2.AspNet", "Fido2.AspNet",
"Duende.IdentityServer", "Duende.IdentityServer",
"Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos",
"Microsoft.Azure.Cosmos.Table",
"Microsoft.Extensions.Caching.StackExchangeRedis", "Microsoft.Extensions.Caching.StackExchangeRedis",
"Microsoft.Extensions.Identity.Stores", "Microsoft.Extensions.Identity.Stores",
"Otp.NET", "Otp.NET",

View File

@ -23,6 +23,7 @@
<PackageReference Include="AspNetCoreRateLimit.Redis" Version="2.0.0" /> <PackageReference Include="AspNetCoreRateLimit.Redis" Version="2.0.0" />
<PackageReference Include="AWSSDK.SimpleEmail" Version="3.7.300.63" /> <PackageReference Include="AWSSDK.SimpleEmail" Version="3.7.300.63" />
<PackageReference Include="AWSSDK.SQS" Version="3.7.300.63" /> <PackageReference Include="AWSSDK.SQS" Version="3.7.300.63" />
<PackageReference Include="Azure.Data.Tables" Version="12.8.3" />
<PackageReference Include="Azure.Extensions.AspNetCore.DataProtection.Blobs" Version="1.3.2" /> <PackageReference Include="Azure.Extensions.AspNetCore.DataProtection.Blobs" Version="1.3.2" />
<PackageReference Include="Azure.Messaging.ServiceBus" Version="7.15.0" /> <PackageReference Include="Azure.Messaging.ServiceBus" Version="7.15.0" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.14.1" /> <PackageReference Include="Azure.Storage.Blobs" Version="12.14.1" />
@ -35,7 +36,6 @@
<PackageReference Include="MailKit" Version="4.4.0" /> <PackageReference Include="MailKit" Version="4.4.0" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.25" /> <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.25" />
<PackageReference Include="Microsoft.Azure.Cosmos" Version="3.38.0" /> <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.38.0" />
<PackageReference Include="Microsoft.Azure.Cosmos.Table" Version="1.0.8" />
<PackageReference Include="Microsoft.Azure.NotificationHubs" Version="4.2.0" /> <PackageReference Include="Microsoft.Azure.NotificationHubs" Version="4.2.0" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.1.5" /> <PackageReference Include="Microsoft.Data.SqlClient" Version="5.1.5" />
<PackageReference Include="Microsoft.Extensions.Caching.Cosmos" Version="1.6.0" /> <PackageReference Include="Microsoft.Extensions.Caching.Cosmos" Version="1.6.0" />

View File

@ -1,134 +0,0 @@
using System.Collections;
using Microsoft.Azure.Cosmos.Table;
namespace Bit.Core.Models.Data;
public class DictionaryEntity : TableEntity, IDictionary<string, EntityProperty>
{
private IDictionary<string, EntityProperty> _properties = new Dictionary<string, EntityProperty>();
public ICollection<EntityProperty> Values => _properties.Values;
public EntityProperty this[string key]
{
get => _properties[key];
set => _properties[key] = value;
}
public int Count => _properties.Count;
public bool IsReadOnly => _properties.IsReadOnly;
public ICollection<string> Keys => _properties.Keys;
public override void ReadEntity(IDictionary<string, EntityProperty> properties,
OperationContext operationContext)
{
_properties = properties;
}
public override IDictionary<string, EntityProperty> WriteEntity(OperationContext operationContext)
{
return _properties;
}
public void Add(string key, EntityProperty value)
{
_properties.Add(key, value);
}
public void Add(string key, bool value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, byte[] value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, DateTime? value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, DateTimeOffset? value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, double value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, Guid value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, int value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, long value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, string value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(KeyValuePair<string, EntityProperty> item)
{
_properties.Add(item);
}
public bool ContainsKey(string key)
{
return _properties.ContainsKey(key);
}
public bool Remove(string key)
{
return _properties.Remove(key);
}
public bool TryGetValue(string key, out EntityProperty value)
{
return _properties.TryGetValue(key, out value);
}
public void Clear()
{
_properties.Clear();
}
public bool Contains(KeyValuePair<string, EntityProperty> item)
{
return _properties.Contains(item);
}
public void CopyTo(KeyValuePair<string, EntityProperty>[] array, int arrayIndex)
{
_properties.CopyTo(array, arrayIndex);
}
public bool Remove(KeyValuePair<string, EntityProperty> item)
{
return _properties.Remove(item);
}
public IEnumerator<KeyValuePair<string, EntityProperty>> GetEnumerator()
{
return _properties.GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return _properties.GetEnumerator();
}
}

View File

@ -1,10 +1,73 @@
using Bit.Core.Enums; using Azure;
using Azure.Data.Tables;
using Bit.Core.Enums;
using Bit.Core.Utilities; using Bit.Core.Utilities;
using Microsoft.Azure.Cosmos.Table;
namespace Bit.Core.Models.Data; namespace Bit.Core.Models.Data;
public class EventTableEntity : TableEntity, IEvent // used solely for interaction with Azure Table Storage
public class AzureEvent : ITableEntity
{
public string PartitionKey { get; set; }
public string RowKey { get; set; }
public DateTimeOffset? Timestamp { get; set; }
public ETag ETag { get; set; }
public DateTime Date { get; set; }
public int Type { get; set; }
public Guid? UserId { get; set; }
public Guid? OrganizationId { get; set; }
public Guid? InstallationId { get; set; }
public Guid? ProviderId { get; set; }
public Guid? CipherId { get; set; }
public Guid? CollectionId { get; set; }
public Guid? PolicyId { get; set; }
public Guid? GroupId { get; set; }
public Guid? OrganizationUserId { get; set; }
public Guid? ProviderUserId { get; set; }
public Guid? ProviderOrganizationId { get; set; }
public int? DeviceType { get; set; }
public string IpAddress { get; set; }
public Guid? ActingUserId { get; set; }
public int? SystemUser { get; set; }
public string DomainName { get; set; }
public Guid? SecretId { get; set; }
public Guid? ServiceAccountId { get; set; }
public EventTableEntity ToEventTableEntity()
{
return new EventTableEntity
{
PartitionKey = PartitionKey,
RowKey = RowKey,
Timestamp = Timestamp,
ETag = ETag,
Date = Date,
Type = (EventType)Type,
UserId = UserId,
OrganizationId = OrganizationId,
InstallationId = InstallationId,
ProviderId = ProviderId,
CipherId = CipherId,
CollectionId = CollectionId,
PolicyId = PolicyId,
GroupId = GroupId,
OrganizationUserId = OrganizationUserId,
ProviderUserId = ProviderUserId,
ProviderOrganizationId = ProviderOrganizationId,
DeviceType = DeviceType.HasValue ? (DeviceType)DeviceType.Value : null,
IpAddress = IpAddress,
ActingUserId = ActingUserId,
SystemUser = SystemUser.HasValue ? (EventSystemUser)SystemUser.Value : null,
DomainName = DomainName,
SecretId = SecretId,
ServiceAccountId = ServiceAccountId
};
}
}
public class EventTableEntity : IEvent
{ {
public EventTableEntity() { } public EventTableEntity() { }
@ -32,6 +95,11 @@ public class EventTableEntity : TableEntity, IEvent
ServiceAccountId = e.ServiceAccountId; ServiceAccountId = e.ServiceAccountId;
} }
public string PartitionKey { get; set; }
public string RowKey { get; set; }
public DateTimeOffset? Timestamp { get; set; }
public ETag ETag { get; set; }
public DateTime Date { get; set; } public DateTime Date { get; set; }
public EventType Type { get; set; } public EventType Type { get; set; }
public Guid? UserId { get; set; } public Guid? UserId { get; set; }
@ -53,65 +121,36 @@ public class EventTableEntity : TableEntity, IEvent
public Guid? SecretId { get; set; } public Guid? SecretId { get; set; }
public Guid? ServiceAccountId { get; set; } public Guid? ServiceAccountId { get; set; }
public override IDictionary<string, EntityProperty> WriteEntity(OperationContext operationContext) public AzureEvent ToAzureEvent()
{ {
var result = base.WriteEntity(operationContext); return new AzureEvent
{
PartitionKey = PartitionKey,
RowKey = RowKey,
Timestamp = Timestamp,
ETag = ETag,
var typeName = nameof(Type); Date = Date,
if (result.ContainsKey(typeName)) Type = (int)Type,
{ UserId = UserId,
result[typeName] = new EntityProperty((int)Type); OrganizationId = OrganizationId,
} InstallationId = InstallationId,
else ProviderId = ProviderId,
{ CipherId = CipherId,
result.Add(typeName, new EntityProperty((int)Type)); CollectionId = CollectionId,
} PolicyId = PolicyId,
GroupId = GroupId,
var deviceTypeName = nameof(DeviceType); OrganizationUserId = OrganizationUserId,
if (result.ContainsKey(deviceTypeName)) ProviderUserId = ProviderUserId,
{ ProviderOrganizationId = ProviderOrganizationId,
result[deviceTypeName] = new EntityProperty((int?)DeviceType); DeviceType = DeviceType.HasValue ? (int)DeviceType.Value : null,
} IpAddress = IpAddress,
else ActingUserId = ActingUserId,
{ SystemUser = SystemUser.HasValue ? (int)SystemUser.Value : null,
result.Add(deviceTypeName, new EntityProperty((int?)DeviceType)); DomainName = DomainName,
} SecretId = SecretId,
ServiceAccountId = ServiceAccountId
var systemUserTypeName = nameof(SystemUser); };
if (result.ContainsKey(systemUserTypeName))
{
result[systemUserTypeName] = new EntityProperty((int?)SystemUser);
}
else
{
result.Add(systemUserTypeName, new EntityProperty((int?)SystemUser));
}
return result;
}
public override void ReadEntity(IDictionary<string, EntityProperty> properties,
OperationContext operationContext)
{
base.ReadEntity(properties, operationContext);
var typeName = nameof(Type);
if (properties.ContainsKey(typeName) && properties[typeName].Int32Value.HasValue)
{
Type = (EventType)properties[typeName].Int32Value.Value;
}
var deviceTypeName = nameof(DeviceType);
if (properties.ContainsKey(deviceTypeName) && properties[deviceTypeName].Int32Value.HasValue)
{
DeviceType = (DeviceType)properties[deviceTypeName].Int32Value.Value;
}
var systemUserTypeName = nameof(SystemUser);
if (properties.ContainsKey(systemUserTypeName) && properties[systemUserTypeName].Int32Value.HasValue)
{
SystemUser = (EventSystemUser)properties[systemUserTypeName].Int32Value.Value;
}
} }
public static List<EventTableEntity> IndexEvent(EventMessage e) public static List<EventTableEntity> IndexEvent(EventMessage e)

View File

@ -1,8 +1,9 @@
using Microsoft.Azure.Cosmos.Table; using Azure;
using Azure.Data.Tables;
namespace Bit.Core.Models.Data; namespace Bit.Core.Models.Data;
public class InstallationDeviceEntity : TableEntity public class InstallationDeviceEntity : ITableEntity
{ {
public InstallationDeviceEntity() { } public InstallationDeviceEntity() { }
@ -27,6 +28,11 @@ public class InstallationDeviceEntity : TableEntity
RowKey = parts[1]; RowKey = parts[1];
} }
public string PartitionKey { get; set; }
public string RowKey { get; set; }
public DateTimeOffset? Timestamp { get; set; }
public ETag ETag { get; set; }
public static bool IsInstallationDeviceId(string deviceId) public static bool IsInstallationDeviceId(string deviceId)
{ {
return deviceId != null && deviceId.Length == 73 && deviceId[36] == '_'; return deviceId != null && deviceId.Length == 73 && deviceId[36] == '_';

View File

@ -1,14 +1,14 @@
using Bit.Core.Models.Data; using Azure.Data.Tables;
using Bit.Core.Models.Data;
using Bit.Core.Settings; using Bit.Core.Settings;
using Bit.Core.Utilities; using Bit.Core.Utilities;
using Bit.Core.Vault.Entities; using Bit.Core.Vault.Entities;
using Microsoft.Azure.Cosmos.Table;
namespace Bit.Core.Repositories.TableStorage; namespace Bit.Core.Repositories.TableStorage;
public class EventRepository : IEventRepository public class EventRepository : IEventRepository
{ {
private readonly CloudTable _table; private readonly TableClient _tableClient;
public EventRepository(GlobalSettings globalSettings) public EventRepository(GlobalSettings globalSettings)
: this(globalSettings.Events.ConnectionString) : this(globalSettings.Events.ConnectionString)
@ -16,9 +16,8 @@ public class EventRepository : IEventRepository
public EventRepository(string storageConnectionString) public EventRepository(string storageConnectionString)
{ {
var storageAccount = CloudStorageAccount.Parse(storageConnectionString); var tableClient = new TableServiceClient(storageConnectionString);
var tableClient = storageAccount.CreateCloudTableClient(); _tableClient = tableClient.GetTableClient("event");
_table = tableClient.GetTableReference("event");
} }
public async Task<PagedResult<IEvent>> GetManyByUserAsync(Guid userId, DateTime startDate, DateTime endDate, public async Task<PagedResult<IEvent>> GetManyByUserAsync(Guid userId, DateTime startDate, DateTime endDate,
@ -76,7 +75,7 @@ public class EventRepository : IEventRepository
throw new ArgumentException(nameof(e)); throw new ArgumentException(nameof(e));
} }
await CreateEntityAsync(entity); await CreateEventAsync(entity);
} }
public async Task CreateManyAsync(IEnumerable<IEvent> e) public async Task CreateManyAsync(IEnumerable<IEvent> e)
@ -99,7 +98,7 @@ public class EventRepository : IEventRepository
var groupEntities = group.ToList(); var groupEntities = group.ToList();
if (groupEntities.Count == 1) if (groupEntities.Count == 1)
{ {
await CreateEntityAsync(groupEntities.First()); await CreateEventAsync(groupEntities.First());
continue; continue;
} }
@ -107,7 +106,7 @@ public class EventRepository : IEventRepository
var iterations = groupEntities.Count / 100; var iterations = groupEntities.Count / 100;
for (var i = 0; i <= iterations; i++) for (var i = 0; i <= iterations; i++)
{ {
var batch = new TableBatchOperation(); var batch = new List<TableTransactionAction>();
var batchEntities = groupEntities.Skip(i * 100).Take(100); var batchEntities = groupEntities.Skip(i * 100).Take(100);
if (!batchEntities.Any()) if (!batchEntities.Any())
{ {
@ -116,19 +115,15 @@ public class EventRepository : IEventRepository
foreach (var entity in batchEntities) foreach (var entity in batchEntities)
{ {
batch.InsertOrReplace(entity); batch.Add(new TableTransactionAction(TableTransactionActionType.Add,
entity.ToAzureEvent()));
} }
await _table.ExecuteBatchAsync(batch); await _tableClient.SubmitTransactionAsync(batch);
} }
} }
} }
public async Task CreateEntityAsync(ITableEntity entity)
{
await _table.ExecuteAsync(TableOperation.InsertOrReplace(entity));
}
public async Task<PagedResult<IEvent>> GetManyAsync(string partitionKey, string rowKey, public async Task<PagedResult<IEvent>> GetManyAsync(string partitionKey, string rowKey,
DateTime startDate, DateTime endDate, PageOptions pageOptions) DateTime startDate, DateTime endDate, PageOptions pageOptions)
{ {
@ -136,60 +131,28 @@ public class EventRepository : IEventRepository
var end = CoreHelpers.DateTimeToTableStorageKey(endDate); var end = CoreHelpers.DateTimeToTableStorageKey(endDate);
var filter = MakeFilter(partitionKey, string.Format(rowKey, start), string.Format(rowKey, end)); var filter = MakeFilter(partitionKey, string.Format(rowKey, start), string.Format(rowKey, end));
var query = new TableQuery<EventTableEntity>().Where(filter).Take(pageOptions.PageSize);
var result = new PagedResult<IEvent>(); var result = new PagedResult<IEvent>();
var continuationToken = DeserializeContinuationToken(pageOptions?.ContinuationToken); var query = _tableClient.QueryAsync<AzureEvent>(filter, pageOptions.PageSize);
var queryResults = await _table.ExecuteQuerySegmentedAsync(query, continuationToken); await using (var enumerator = query.AsPages(pageOptions?.ContinuationToken,
result.ContinuationToken = SerializeContinuationToken(queryResults.ContinuationToken); pageOptions.PageSize).GetAsyncEnumerator())
result.Data.AddRange(queryResults.Results); {
await enumerator.MoveNextAsync();
result.ContinuationToken = enumerator.Current.ContinuationToken;
result.Data.AddRange(enumerator.Current.Values.Select(e => e.ToEventTableEntity()));
}
return result; return result;
} }
private async Task CreateEventAsync(EventTableEntity entity)
{
await _tableClient.UpsertEntityAsync(entity.ToAzureEvent());
}
private string MakeFilter(string partitionKey, string rowStart, string rowEnd) private string MakeFilter(string partitionKey, string rowStart, string rowEnd)
{ {
var rowFilter = TableQuery.CombineFilters( return $"PartitionKey eq '{partitionKey}' and RowKey le '{rowStart}' and RowKey ge '{rowEnd}'";
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, $"{rowStart}`"),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, $"{rowEnd}_"));
return TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey),
TableOperators.And,
rowFilter);
}
private string SerializeContinuationToken(TableContinuationToken token)
{
if (token == null)
{
return null;
}
return string.Format("{0}__{1}__{2}__{3}", (int)token.TargetLocation, token.NextTableName,
token.NextPartitionKey, token.NextRowKey);
}
private TableContinuationToken DeserializeContinuationToken(string token)
{
if (string.IsNullOrWhiteSpace(token))
{
return null;
}
var tokenParts = token.Split(new string[] { "__" }, StringSplitOptions.None);
if (tokenParts.Length < 4 || !Enum.TryParse(tokenParts[0], out StorageLocation tLoc))
{
return null;
}
return new TableContinuationToken
{
TargetLocation = tLoc,
NextTableName = string.IsNullOrWhiteSpace(tokenParts[1]) ? null : tokenParts[1],
NextPartitionKey = string.IsNullOrWhiteSpace(tokenParts[2]) ? null : tokenParts[2],
NextRowKey = string.IsNullOrWhiteSpace(tokenParts[3]) ? null : tokenParts[3]
};
} }
} }

View File

@ -1,13 +1,12 @@
using System.Net; using Azure.Data.Tables;
using Bit.Core.Models.Data; using Bit.Core.Models.Data;
using Bit.Core.Settings; using Bit.Core.Settings;
using Microsoft.Azure.Cosmos.Table;
namespace Bit.Core.Repositories.TableStorage; namespace Bit.Core.Repositories.TableStorage;
public class InstallationDeviceRepository : IInstallationDeviceRepository public class InstallationDeviceRepository : IInstallationDeviceRepository
{ {
private readonly CloudTable _table; private readonly TableClient _tableClient;
public InstallationDeviceRepository(GlobalSettings globalSettings) public InstallationDeviceRepository(GlobalSettings globalSettings)
: this(globalSettings.Events.ConnectionString) : this(globalSettings.Events.ConnectionString)
@ -15,14 +14,13 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository
public InstallationDeviceRepository(string storageConnectionString) public InstallationDeviceRepository(string storageConnectionString)
{ {
var storageAccount = CloudStorageAccount.Parse(storageConnectionString); var tableClient = new TableServiceClient(storageConnectionString);
var tableClient = storageAccount.CreateCloudTableClient(); _tableClient = tableClient.GetTableClient("installationdevice");
_table = tableClient.GetTableReference("installationdevice");
} }
public async Task UpsertAsync(InstallationDeviceEntity entity) public async Task UpsertAsync(InstallationDeviceEntity entity)
{ {
await _table.ExecuteAsync(TableOperation.InsertOrReplace(entity)); await _tableClient.UpsertEntityAsync(entity);
} }
public async Task UpsertManyAsync(IList<InstallationDeviceEntity> entities) public async Task UpsertManyAsync(IList<InstallationDeviceEntity> entities)
@ -52,7 +50,7 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository
var iterations = groupEntities.Count / 100; var iterations = groupEntities.Count / 100;
for (var i = 0; i <= iterations; i++) for (var i = 0; i <= iterations; i++)
{ {
var batch = new TableBatchOperation(); var batch = new List<TableTransactionAction>();
var batchEntities = groupEntities.Skip(i * 100).Take(100); var batchEntities = groupEntities.Skip(i * 100).Take(100);
if (!batchEntities.Any()) if (!batchEntities.Any())
{ {
@ -61,24 +59,16 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository
foreach (var entity in batchEntities) foreach (var entity in batchEntities)
{ {
batch.InsertOrReplace(entity); batch.Add(new TableTransactionAction(TableTransactionActionType.UpsertReplace, entity));
} }
await _table.ExecuteBatchAsync(batch); await _tableClient.SubmitTransactionAsync(batch);
} }
} }
} }
public async Task DeleteAsync(InstallationDeviceEntity entity) public async Task DeleteAsync(InstallationDeviceEntity entity)
{ {
try await _tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey);
{
entity.ETag = "*";
await _table.ExecuteAsync(TableOperation.Delete(entity));
}
catch (StorageException e) when (e.RequestInformation.HttpStatusCode != (int)HttpStatusCode.NotFound)
{
throw;
}
} }
} }