Upstream/bitwarden-server
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2025-01-08 19:47:44 +01:00
Code Issues Projects Releases Wiki Activity

[PM-6762] Move to Azure.Data.Tables (#3888)

Browse Source 
* Move to Azure.Data.Tables

* Reorder usings

* Add new package to Renovate

* Add manual serialization and deserialization due to enums

* Properly retrieve just the next page
This commit is contained in:
Matt Bishop 2024-03-28 16:36:24 -04:00 committed by GitHub
parent ffd988eeda
commit c53e5eeab3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 144 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/renovate.json vendored
View File

@ -44,6 +44,7 @@
"matchPackageNames": [
"AspNetCoreRateLimit",
"AspNetCoreRateLimit.Redis",
"Azure.Data.Tables",
"Azure.Extensions.AspNetCore.DataProtection.Blobs",
"Azure.Messaging.EventGrid",
"Azure.Messaging.ServiceBus",
@ -53,7 +54,6 @@
"Fido2.AspNet",
"Duende.IdentityServer",
"Microsoft.Azure.Cosmos",
"Microsoft.Azure.Cosmos.Table",
"Microsoft.Extensions.Caching.StackExchangeRedis",
"Microsoft.Extensions.Identity.Stores",
"Otp.NET",

2
src/Core/Core.csproj
View File

@ -23,6 +23,7 @@
<PackageReference Include="AspNetCoreRateLimit.Redis" Version="2.0.0" />
<PackageReference Include="AWSSDK.SimpleEmail" Version="3.7.300.63" />
<PackageReference Include="AWSSDK.SQS" Version="3.7.300.63" />
<PackageReference Include="Azure.Data.Tables" Version="12.8.3" />
<PackageReference Include="Azure.Extensions.AspNetCore.DataProtection.Blobs" Version="1.3.2" />
<PackageReference Include="Azure.Messaging.ServiceBus" Version="7.15.0" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.14.1" />
@ -35,7 +36,6 @@
<PackageReference Include="MailKit" Version="4.4.0" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.25" />
<PackageReference Include="Microsoft.Azure.Cosmos" Version="3.38.0" />
<PackageReference Include="Microsoft.Azure.Cosmos.Table" Version="1.0.8" />
<PackageReference Include="Microsoft.Azure.NotificationHubs" Version="4.2.0" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.1.5" />
<PackageReference Include="Microsoft.Extensions.Caching.Cosmos" Version="1.6.0" />

134
src/Core/Models/Data/DictionaryEntity.cs
View File

@ -1,134 +0,0 @@
using System.Collections;
using Microsoft.Azure.Cosmos.Table;
namespace Bit.Core.Models.Data;
public class DictionaryEntity : TableEntity, IDictionary<string, EntityProperty>
{
private IDictionary<string, EntityProperty> _properties = new Dictionary<string, EntityProperty>();
public ICollection<EntityProperty> Values => _properties.Values;
public EntityProperty this[string key]
{
get => _properties[key];
set => _properties[key] = value;
}
public int Count => _properties.Count;
public bool IsReadOnly => _properties.IsReadOnly;
public ICollection<string> Keys => _properties.Keys;
public override void ReadEntity(IDictionary<string, EntityProperty> properties,
OperationContext operationContext)
{
_properties = properties;
}
public override IDictionary<string, EntityProperty> WriteEntity(OperationContext operationContext)
{
return _properties;
}
public void Add(string key, EntityProperty value)
{
_properties.Add(key, value);
}
public void Add(string key, bool value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, byte[] value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, DateTime? value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, DateTimeOffset? value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, double value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, Guid value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, int value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, long value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(string key, string value)
{
_properties.Add(key, new EntityProperty(value));
}
public void Add(KeyValuePair<string, EntityProperty> item)
{
_properties.Add(item);
}
public bool ContainsKey(string key)
{
return _properties.ContainsKey(key);
}
public bool Remove(string key)
{
return _properties.Remove(key);
}
public bool TryGetValue(string key, out EntityProperty value)
{
return _properties.TryGetValue(key, out value);
}
public void Clear()
{
_properties.Clear();
}
public bool Contains(KeyValuePair<string, EntityProperty> item)
{
return _properties.Contains(item);
}
public void CopyTo(KeyValuePair<string, EntityProperty>[] array, int arrayIndex)
{
_properties.CopyTo(array, arrayIndex);
}
public bool Remove(KeyValuePair<string, EntityProperty> item)
{
return _properties.Remove(item);
}
public IEnumerator<KeyValuePair<string, EntityProperty>> GetEnumerator()
{
return _properties.GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return _properties.GetEnumerator();
}
}

159
src/Core/Models/Data/EventTableEntity.cs
View File

@ -1,10 +1,73 @@
using Bit.Core.Enums;
using Azure;
using Azure.Data.Tables;
using Bit.Core.Enums;
using Bit.Core.Utilities;
using Microsoft.Azure.Cosmos.Table;
namespace Bit.Core.Models.Data;
public class EventTableEntity : TableEntity, IEvent
// used solely for interaction with Azure Table Storage
public class AzureEvent : ITableEntity
{
public string PartitionKey { get; set; }
public string RowKey { get; set; }
public DateTimeOffset? Timestamp { get; set; }
public ETag ETag { get; set; }
public DateTime Date { get; set; }
public int Type { get; set; }
public Guid? UserId { get; set; }
public Guid? OrganizationId { get; set; }
public Guid? InstallationId { get; set; }
public Guid? ProviderId { get; set; }
public Guid? CipherId { get; set; }
public Guid? CollectionId { get; set; }
public Guid? PolicyId { get; set; }
public Guid? GroupId { get; set; }
public Guid? OrganizationUserId { get; set; }
public Guid? ProviderUserId { get; set; }
public Guid? ProviderOrganizationId { get; set; }
public int? DeviceType { get; set; }
public string IpAddress { get; set; }
public Guid? ActingUserId { get; set; }
public int? SystemUser { get; set; }
public string DomainName { get; set; }
public Guid? SecretId { get; set; }
public Guid? ServiceAccountId { get; set; }
public EventTableEntity ToEventTableEntity()
{
return new EventTableEntity
{
PartitionKey = PartitionKey,
RowKey = RowKey,
Timestamp = Timestamp,
ETag = ETag,
Date = Date,
Type = (EventType)Type,
UserId = UserId,
OrganizationId = OrganizationId,
InstallationId = InstallationId,
ProviderId = ProviderId,
CipherId = CipherId,
CollectionId = CollectionId,
PolicyId = PolicyId,
GroupId = GroupId,
OrganizationUserId = OrganizationUserId,
ProviderUserId = ProviderUserId,
ProviderOrganizationId = ProviderOrganizationId,
DeviceType = DeviceType.HasValue ? (DeviceType)DeviceType.Value : null,
IpAddress = IpAddress,
ActingUserId = ActingUserId,
SystemUser = SystemUser.HasValue ? (EventSystemUser)SystemUser.Value : null,
DomainName = DomainName,
SecretId = SecretId,
ServiceAccountId = ServiceAccountId
};
}
}
public class EventTableEntity : IEvent
{
public EventTableEntity() { }
@ -32,6 +95,11 @@ public class EventTableEntity : TableEntity, IEvent
ServiceAccountId = e.ServiceAccountId;
}
public string PartitionKey { get; set; }
public string RowKey { get; set; }
public DateTimeOffset? Timestamp { get; set; }
public ETag ETag { get; set; }
public DateTime Date { get; set; }
public EventType Type { get; set; }
public Guid? UserId { get; set; }
@ -53,65 +121,36 @@ public class EventTableEntity : TableEntity, IEvent
public Guid? SecretId { get; set; }
public Guid? ServiceAccountId { get; set; }
public override IDictionary<string, EntityProperty> WriteEntity(OperationContext operationContext)
public AzureEvent ToAzureEvent()
{
var result = base.WriteEntity(operationContext);
return new AzureEvent
{
PartitionKey = PartitionKey,
RowKey = RowKey,
Timestamp = Timestamp,
ETag = ETag,
var typeName = nameof(Type);
if (result.ContainsKey(typeName))
{
result[typeName] = new EntityProperty((int)Type);
}
else
{
result.Add(typeName, new EntityProperty((int)Type));
}
var deviceTypeName = nameof(DeviceType);
if (result.ContainsKey(deviceTypeName))
{
result[deviceTypeName] = new EntityProperty((int?)DeviceType);
}
else
{
result.Add(deviceTypeName, new EntityProperty((int?)DeviceType));
}
var systemUserTypeName = nameof(SystemUser);
if (result.ContainsKey(systemUserTypeName))
{
result[systemUserTypeName] = new EntityProperty((int?)SystemUser);
}
else
{
result.Add(systemUserTypeName, new EntityProperty((int?)SystemUser));
}
return result;
}
public override void ReadEntity(IDictionary<string, EntityProperty> properties,
OperationContext operationContext)
{
base.ReadEntity(properties, operationContext);
var typeName = nameof(Type);
if (properties.ContainsKey(typeName) && properties[typeName].Int32Value.HasValue)
{
Type = (EventType)properties[typeName].Int32Value.Value;
}
var deviceTypeName = nameof(DeviceType);
if (properties.ContainsKey(deviceTypeName) && properties[deviceTypeName].Int32Value.HasValue)
{
DeviceType = (DeviceType)properties[deviceTypeName].Int32Value.Value;
}
var systemUserTypeName = nameof(SystemUser);
if (properties.ContainsKey(systemUserTypeName) && properties[systemUserTypeName].Int32Value.HasValue)
{
SystemUser = (EventSystemUser)properties[systemUserTypeName].Int32Value.Value;
}
Date = Date,
Type = (int)Type,
UserId = UserId,
OrganizationId = OrganizationId,
InstallationId = InstallationId,
ProviderId = ProviderId,
CipherId = CipherId,
CollectionId = CollectionId,
PolicyId = PolicyId,
GroupId = GroupId,
OrganizationUserId = OrganizationUserId,
ProviderUserId = ProviderUserId,
ProviderOrganizationId = ProviderOrganizationId,
DeviceType = DeviceType.HasValue ? (int)DeviceType.Value : null,
IpAddress = IpAddress,
ActingUserId = ActingUserId,
SystemUser = SystemUser.HasValue ? (int)SystemUser.Value : null,
DomainName = DomainName,
SecretId = SecretId,
ServiceAccountId = ServiceAccountId
};
}
public static List<EventTableEntity> IndexEvent(EventMessage e)

10
src/Core/Models/Data/InstallationDeviceEntity.cs
View File

@ -1,8 +1,9 @@
using Microsoft.Azure.Cosmos.Table;
using Azure;
using Azure.Data.Tables;
namespace Bit.Core.Models.Data;
public class InstallationDeviceEntity : TableEntity
public class InstallationDeviceEntity : ITableEntity
{
public InstallationDeviceEntity() { }
@ -27,6 +28,11 @@ public class InstallationDeviceEntity : TableEntity
RowKey = parts[1];
}
public string PartitionKey { get; set; }
public string RowKey { get; set; }
public DateTimeOffset? Timestamp { get; set; }
public ETag ETag { get; set; }
public static bool IsInstallationDeviceId(string deviceId)
{
return deviceId != null && deviceId.Length == 73 && deviceId[36] == '_';

89
src/Core/Repositories/TableStorage/EventRepository.cs
View File

@ -1,14 +1,14 @@
using Bit.Core.Models.Data;
using Azure.Data.Tables;
using Bit.Core.Models.Data;
using Bit.Core.Settings;
using Bit.Core.Utilities;
using Bit.Core.Vault.Entities;
using Microsoft.Azure.Cosmos.Table;
namespace Bit.Core.Repositories.TableStorage;
public class EventRepository : IEventRepository
{
private readonly CloudTable _table;
private readonly TableClient _tableClient;
public EventRepository(GlobalSettings globalSettings)
: this(globalSettings.Events.ConnectionString)
@ -16,9 +16,8 @@ public class EventRepository : IEventRepository
public EventRepository(string storageConnectionString)
{
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var tableClient = storageAccount.CreateCloudTableClient();
_table = tableClient.GetTableReference("event");
var tableClient = new TableServiceClient(storageConnectionString);
_tableClient = tableClient.GetTableClient("event");
}
public async Task<PagedResult<IEvent>> GetManyByUserAsync(Guid userId, DateTime startDate, DateTime endDate,
@ -76,7 +75,7 @@ public class EventRepository : IEventRepository
throw new ArgumentException(nameof(e));
}
await CreateEntityAsync(entity);
await CreateEventAsync(entity);
}
public async Task CreateManyAsync(IEnumerable<IEvent> e)
@ -99,7 +98,7 @@ public class EventRepository : IEventRepository
var groupEntities = group.ToList();
if (groupEntities.Count == 1)
{
await CreateEntityAsync(groupEntities.First());
await CreateEventAsync(groupEntities.First());
continue;
}
@ -107,7 +106,7 @@ public class EventRepository : IEventRepository
var iterations = groupEntities.Count / 100;
for (var i = 0; i <= iterations; i++)
{
var batch = new TableBatchOperation();
var batch = new List<TableTransactionAction>();
var batchEntities = groupEntities.Skip(i * 100).Take(100);
if (!batchEntities.Any())
{
@ -116,19 +115,15 @@ public class EventRepository : IEventRepository
foreach (var entity in batchEntities)
{
batch.InsertOrReplace(entity);
batch.Add(new TableTransactionAction(TableTransactionActionType.Add,
entity.ToAzureEvent()));
}
await _table.ExecuteBatchAsync(batch);
await _tableClient.SubmitTransactionAsync(batch);
}
}
}
public async Task CreateEntityAsync(ITableEntity entity)
{
await _table.ExecuteAsync(TableOperation.InsertOrReplace(entity));
}
public async Task<PagedResult<IEvent>> GetManyAsync(string partitionKey, string rowKey,
DateTime startDate, DateTime endDate, PageOptions pageOptions)
{
@ -136,60 +131,28 @@ public class EventRepository : IEventRepository
var end = CoreHelpers.DateTimeToTableStorageKey(endDate);
var filter = MakeFilter(partitionKey, string.Format(rowKey, start), string.Format(rowKey, end));
var query = new TableQuery<EventTableEntity>().Where(filter).Take(pageOptions.PageSize);
var result = new PagedResult<IEvent>();
var continuationToken = DeserializeContinuationToken(pageOptions?.ContinuationToken);
var query = _tableClient.QueryAsync<AzureEvent>(filter, pageOptions.PageSize);
var queryResults = await _table.ExecuteQuerySegmentedAsync(query, continuationToken);
result.ContinuationToken = SerializeContinuationToken(queryResults.ContinuationToken);
result.Data.AddRange(queryResults.Results);
await using (var enumerator = query.AsPages(pageOptions?.ContinuationToken,
pageOptions.PageSize).GetAsyncEnumerator())
{
await enumerator.MoveNextAsync();
result.ContinuationToken = enumerator.Current.ContinuationToken;
result.Data.AddRange(enumerator.Current.Values.Select(e => e.ToEventTableEntity()));
}
return result;
}
private async Task CreateEventAsync(EventTableEntity entity)
{
await _tableClient.UpsertEntityAsync(entity.ToAzureEvent());
}
private string MakeFilter(string partitionKey, string rowStart, string rowEnd)
{
var rowFilter = TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, $"{rowStart}`"),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, $"{rowEnd}_"));
return TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey),
TableOperators.And,
rowFilter);
}
private string SerializeContinuationToken(TableContinuationToken token)
{
if (token == null)
{
return null;
}
return string.Format("{0}__{1}__{2}__{3}", (int)token.TargetLocation, token.NextTableName,
token.NextPartitionKey, token.NextRowKey);
}
private TableContinuationToken DeserializeContinuationToken(string token)
{
if (string.IsNullOrWhiteSpace(token))
{
return null;
}
var tokenParts = token.Split(new string[] { "__" }, StringSplitOptions.None);
if (tokenParts.Length < 4 || !Enum.TryParse(tokenParts[0], out StorageLocation tLoc))
{
return null;
}
return new TableContinuationToken
{
TargetLocation = tLoc,
NextTableName = string.IsNullOrWhiteSpace(tokenParts[1]) ? null : tokenParts[1],
NextPartitionKey = string.IsNullOrWhiteSpace(tokenParts[2]) ? null : tokenParts[2],
NextRowKey = string.IsNullOrWhiteSpace(tokenParts[3]) ? null : tokenParts[3]
};
return $"PartitionKey eq '{partitionKey}' and RowKey le '{rowStart}' and RowKey ge '{rowEnd}'";
}
}

28
src/Core/Repositories/TableStorage/InstallationDeviceRepository.cs
View File

@ -1,13 +1,12 @@
using System.Net;
using Azure.Data.Tables;
using Bit.Core.Models.Data;
using Bit.Core.Settings;
using Microsoft.Azure.Cosmos.Table;
namespace Bit.Core.Repositories.TableStorage;
public class InstallationDeviceRepository : IInstallationDeviceRepository
{
private readonly CloudTable _table;
private readonly TableClient _tableClient;
public InstallationDeviceRepository(GlobalSettings globalSettings)
: this(globalSettings.Events.ConnectionString)
@ -15,14 +14,13 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository
public InstallationDeviceRepository(string storageConnectionString)
{
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var tableClient = storageAccount.CreateCloudTableClient();
_table = tableClient.GetTableReference("installationdevice");
var tableClient = new TableServiceClient(storageConnectionString);
_tableClient = tableClient.GetTableClient("installationdevice");
}
public async Task UpsertAsync(InstallationDeviceEntity entity)
{
await _table.ExecuteAsync(TableOperation.InsertOrReplace(entity));
await _tableClient.UpsertEntityAsync(entity);
}
public async Task UpsertManyAsync(IList<InstallationDeviceEntity> entities)
@ -52,7 +50,7 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository
var iterations = groupEntities.Count / 100;
for (var i = 0; i <= iterations; i++)
{
var batch = new TableBatchOperation();
var batch = new List<TableTransactionAction>();
var batchEntities = groupEntities.Skip(i * 100).Take(100);
if (!batchEntities.Any())
{
@ -61,24 +59,16 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository
foreach (var entity in batchEntities)
{
batch.InsertOrReplace(entity);
batch.Add(new TableTransactionAction(TableTransactionActionType.UpsertReplace, entity));
}
await _table.ExecuteBatchAsync(batch);
await _tableClient.SubmitTransactionAsync(batch);
}
}
}
public async Task DeleteAsync(InstallationDeviceEntity entity)
{
try
{
entity.ETag = "*";
await _table.ExecuteAsync(TableOperation.Delete(entity));
}
catch (StorageException e) when (e.RequestInformation.HttpStatusCode != (int)HttpStatusCode.NotFound)
{
throw;
}
await _tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey);
}
}