diff --git a/.github/renovate.json b/.github/renovate.json index 18d6e0bb61..91774ca33e 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -44,6 +44,7 @@ "matchPackageNames": [ "AspNetCoreRateLimit", "AspNetCoreRateLimit.Redis", + "Azure.Data.Tables", "Azure.Extensions.AspNetCore.DataProtection.Blobs", "Azure.Messaging.EventGrid", "Azure.Messaging.ServiceBus", @@ -53,7 +54,6 @@ "Fido2.AspNet", "Duende.IdentityServer", "Microsoft.Azure.Cosmos", - "Microsoft.Azure.Cosmos.Table", "Microsoft.Extensions.Caching.StackExchangeRedis", "Microsoft.Extensions.Identity.Stores", "Otp.NET", diff --git a/src/Core/Core.csproj b/src/Core/Core.csproj index 92c2198242..4189b9f525 100644 --- a/src/Core/Core.csproj +++ b/src/Core/Core.csproj @@ -23,6 +23,7 @@ + @@ -35,7 +36,6 @@ - diff --git a/src/Core/Models/Data/DictionaryEntity.cs b/src/Core/Models/Data/DictionaryEntity.cs deleted file mode 100644 index 72e6c871c7..0000000000 --- a/src/Core/Models/Data/DictionaryEntity.cs +++ /dev/null @@ -1,134 +0,0 @@ -using System.Collections; -using Microsoft.Azure.Cosmos.Table; - -namespace Bit.Core.Models.Data; - -public class DictionaryEntity : TableEntity, IDictionary -{ - private IDictionary _properties = new Dictionary(); - - public ICollection Values => _properties.Values; - - public EntityProperty this[string key] - { - get => _properties[key]; - set => _properties[key] = value; - } - - public int Count => _properties.Count; - - public bool IsReadOnly => _properties.IsReadOnly; - - public ICollection Keys => _properties.Keys; - - public override void ReadEntity(IDictionary properties, - OperationContext operationContext) - { - _properties = properties; - } - - public override IDictionary WriteEntity(OperationContext operationContext) - { - return _properties; - } - - public void Add(string key, EntityProperty value) - { - _properties.Add(key, value); - } - - public void Add(string key, bool value) - { - _properties.Add(key, new EntityProperty(value)); - } - - public void Add(string key, byte[] value) - { - _properties.Add(key, new EntityProperty(value)); - } - - public void Add(string key, DateTime? value) - { - _properties.Add(key, new EntityProperty(value)); - } - - public void Add(string key, DateTimeOffset? value) - { - _properties.Add(key, new EntityProperty(value)); - } - - public void Add(string key, double value) - { - _properties.Add(key, new EntityProperty(value)); - } - - public void Add(string key, Guid value) - { - _properties.Add(key, new EntityProperty(value)); - } - - public void Add(string key, int value) - { - _properties.Add(key, new EntityProperty(value)); - } - - public void Add(string key, long value) - { - _properties.Add(key, new EntityProperty(value)); - } - - public void Add(string key, string value) - { - _properties.Add(key, new EntityProperty(value)); - } - - public void Add(KeyValuePair item) - { - _properties.Add(item); - } - - public bool ContainsKey(string key) - { - return _properties.ContainsKey(key); - } - - public bool Remove(string key) - { - return _properties.Remove(key); - } - - public bool TryGetValue(string key, out EntityProperty value) - { - return _properties.TryGetValue(key, out value); - } - - public void Clear() - { - _properties.Clear(); - } - - public bool Contains(KeyValuePair item) - { - return _properties.Contains(item); - } - - public void CopyTo(KeyValuePair[] array, int arrayIndex) - { - _properties.CopyTo(array, arrayIndex); - } - - public bool Remove(KeyValuePair item) - { - return _properties.Remove(item); - } - - public IEnumerator> GetEnumerator() - { - return _properties.GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return _properties.GetEnumerator(); - } -} diff --git a/src/Core/Models/Data/EventTableEntity.cs b/src/Core/Models/Data/EventTableEntity.cs index df4a85acaf..69365f4127 100644 --- a/src/Core/Models/Data/EventTableEntity.cs +++ b/src/Core/Models/Data/EventTableEntity.cs @@ -1,10 +1,73 @@ -using Bit.Core.Enums; +using Azure; +using Azure.Data.Tables; +using Bit.Core.Enums; using Bit.Core.Utilities; -using Microsoft.Azure.Cosmos.Table; namespace Bit.Core.Models.Data; -public class EventTableEntity : TableEntity, IEvent +// used solely for interaction with Azure Table Storage +public class AzureEvent : ITableEntity +{ + public string PartitionKey { get; set; } + public string RowKey { get; set; } + public DateTimeOffset? Timestamp { get; set; } + public ETag ETag { get; set; } + + public DateTime Date { get; set; } + public int Type { get; set; } + public Guid? UserId { get; set; } + public Guid? OrganizationId { get; set; } + public Guid? InstallationId { get; set; } + public Guid? ProviderId { get; set; } + public Guid? CipherId { get; set; } + public Guid? CollectionId { get; set; } + public Guid? PolicyId { get; set; } + public Guid? GroupId { get; set; } + public Guid? OrganizationUserId { get; set; } + public Guid? ProviderUserId { get; set; } + public Guid? ProviderOrganizationId { get; set; } + public int? DeviceType { get; set; } + public string IpAddress { get; set; } + public Guid? ActingUserId { get; set; } + public int? SystemUser { get; set; } + public string DomainName { get; set; } + public Guid? SecretId { get; set; } + public Guid? ServiceAccountId { get; set; } + + public EventTableEntity ToEventTableEntity() + { + return new EventTableEntity + { + PartitionKey = PartitionKey, + RowKey = RowKey, + Timestamp = Timestamp, + ETag = ETag, + + Date = Date, + Type = (EventType)Type, + UserId = UserId, + OrganizationId = OrganizationId, + InstallationId = InstallationId, + ProviderId = ProviderId, + CipherId = CipherId, + CollectionId = CollectionId, + PolicyId = PolicyId, + GroupId = GroupId, + OrganizationUserId = OrganizationUserId, + ProviderUserId = ProviderUserId, + ProviderOrganizationId = ProviderOrganizationId, + DeviceType = DeviceType.HasValue ? (DeviceType)DeviceType.Value : null, + IpAddress = IpAddress, + ActingUserId = ActingUserId, + SystemUser = SystemUser.HasValue ? (EventSystemUser)SystemUser.Value : null, + DomainName = DomainName, + SecretId = SecretId, + ServiceAccountId = ServiceAccountId + }; + } +} + +public class EventTableEntity : IEvent { public EventTableEntity() { } @@ -32,6 +95,11 @@ public class EventTableEntity : TableEntity, IEvent ServiceAccountId = e.ServiceAccountId; } + public string PartitionKey { get; set; } + public string RowKey { get; set; } + public DateTimeOffset? Timestamp { get; set; } + public ETag ETag { get; set; } + public DateTime Date { get; set; } public EventType Type { get; set; } public Guid? UserId { get; set; } @@ -53,65 +121,36 @@ public class EventTableEntity : TableEntity, IEvent public Guid? SecretId { get; set; } public Guid? ServiceAccountId { get; set; } - public override IDictionary WriteEntity(OperationContext operationContext) + public AzureEvent ToAzureEvent() { - var result = base.WriteEntity(operationContext); + return new AzureEvent + { + PartitionKey = PartitionKey, + RowKey = RowKey, + Timestamp = Timestamp, + ETag = ETag, - var typeName = nameof(Type); - if (result.ContainsKey(typeName)) - { - result[typeName] = new EntityProperty((int)Type); - } - else - { - result.Add(typeName, new EntityProperty((int)Type)); - } - - var deviceTypeName = nameof(DeviceType); - if (result.ContainsKey(deviceTypeName)) - { - result[deviceTypeName] = new EntityProperty((int?)DeviceType); - } - else - { - result.Add(deviceTypeName, new EntityProperty((int?)DeviceType)); - } - - var systemUserTypeName = nameof(SystemUser); - if (result.ContainsKey(systemUserTypeName)) - { - result[systemUserTypeName] = new EntityProperty((int?)SystemUser); - } - else - { - result.Add(systemUserTypeName, new EntityProperty((int?)SystemUser)); - } - - return result; - } - - public override void ReadEntity(IDictionary properties, - OperationContext operationContext) - { - base.ReadEntity(properties, operationContext); - - var typeName = nameof(Type); - if (properties.ContainsKey(typeName) && properties[typeName].Int32Value.HasValue) - { - Type = (EventType)properties[typeName].Int32Value.Value; - } - - var deviceTypeName = nameof(DeviceType); - if (properties.ContainsKey(deviceTypeName) && properties[deviceTypeName].Int32Value.HasValue) - { - DeviceType = (DeviceType)properties[deviceTypeName].Int32Value.Value; - } - - var systemUserTypeName = nameof(SystemUser); - if (properties.ContainsKey(systemUserTypeName) && properties[systemUserTypeName].Int32Value.HasValue) - { - SystemUser = (EventSystemUser)properties[systemUserTypeName].Int32Value.Value; - } + Date = Date, + Type = (int)Type, + UserId = UserId, + OrganizationId = OrganizationId, + InstallationId = InstallationId, + ProviderId = ProviderId, + CipherId = CipherId, + CollectionId = CollectionId, + PolicyId = PolicyId, + GroupId = GroupId, + OrganizationUserId = OrganizationUserId, + ProviderUserId = ProviderUserId, + ProviderOrganizationId = ProviderOrganizationId, + DeviceType = DeviceType.HasValue ? (int)DeviceType.Value : null, + IpAddress = IpAddress, + ActingUserId = ActingUserId, + SystemUser = SystemUser.HasValue ? (int)SystemUser.Value : null, + DomainName = DomainName, + SecretId = SecretId, + ServiceAccountId = ServiceAccountId + }; } public static List IndexEvent(EventMessage e) diff --git a/src/Core/Models/Data/InstallationDeviceEntity.cs b/src/Core/Models/Data/InstallationDeviceEntity.cs index cb7bf00873..3186efc661 100644 --- a/src/Core/Models/Data/InstallationDeviceEntity.cs +++ b/src/Core/Models/Data/InstallationDeviceEntity.cs @@ -1,8 +1,9 @@ -using Microsoft.Azure.Cosmos.Table; +using Azure; +using Azure.Data.Tables; namespace Bit.Core.Models.Data; -public class InstallationDeviceEntity : TableEntity +public class InstallationDeviceEntity : ITableEntity { public InstallationDeviceEntity() { } @@ -27,6 +28,11 @@ public class InstallationDeviceEntity : TableEntity RowKey = parts[1]; } + public string PartitionKey { get; set; } + public string RowKey { get; set; } + public DateTimeOffset? Timestamp { get; set; } + public ETag ETag { get; set; } + public static bool IsInstallationDeviceId(string deviceId) { return deviceId != null && deviceId.Length == 73 && deviceId[36] == '_'; diff --git a/src/Core/Repositories/TableStorage/EventRepository.cs b/src/Core/Repositories/TableStorage/EventRepository.cs index 7044850033..7c5cb97dba 100644 --- a/src/Core/Repositories/TableStorage/EventRepository.cs +++ b/src/Core/Repositories/TableStorage/EventRepository.cs @@ -1,14 +1,14 @@ -using Bit.Core.Models.Data; +using Azure.Data.Tables; +using Bit.Core.Models.Data; using Bit.Core.Settings; using Bit.Core.Utilities; using Bit.Core.Vault.Entities; -using Microsoft.Azure.Cosmos.Table; namespace Bit.Core.Repositories.TableStorage; public class EventRepository : IEventRepository { - private readonly CloudTable _table; + private readonly TableClient _tableClient; public EventRepository(GlobalSettings globalSettings) : this(globalSettings.Events.ConnectionString) @@ -16,9 +16,8 @@ public class EventRepository : IEventRepository public EventRepository(string storageConnectionString) { - var storageAccount = CloudStorageAccount.Parse(storageConnectionString); - var tableClient = storageAccount.CreateCloudTableClient(); - _table = tableClient.GetTableReference("event"); + var tableClient = new TableServiceClient(storageConnectionString); + _tableClient = tableClient.GetTableClient("event"); } public async Task> GetManyByUserAsync(Guid userId, DateTime startDate, DateTime endDate, @@ -76,7 +75,7 @@ public class EventRepository : IEventRepository throw new ArgumentException(nameof(e)); } - await CreateEntityAsync(entity); + await CreateEventAsync(entity); } public async Task CreateManyAsync(IEnumerable e) @@ -99,7 +98,7 @@ public class EventRepository : IEventRepository var groupEntities = group.ToList(); if (groupEntities.Count == 1) { - await CreateEntityAsync(groupEntities.First()); + await CreateEventAsync(groupEntities.First()); continue; } @@ -107,7 +106,7 @@ public class EventRepository : IEventRepository var iterations = groupEntities.Count / 100; for (var i = 0; i <= iterations; i++) { - var batch = new TableBatchOperation(); + var batch = new List(); var batchEntities = groupEntities.Skip(i * 100).Take(100); if (!batchEntities.Any()) { @@ -116,19 +115,15 @@ public class EventRepository : IEventRepository foreach (var entity in batchEntities) { - batch.InsertOrReplace(entity); + batch.Add(new TableTransactionAction(TableTransactionActionType.Add, + entity.ToAzureEvent())); } - await _table.ExecuteBatchAsync(batch); + await _tableClient.SubmitTransactionAsync(batch); } } } - public async Task CreateEntityAsync(ITableEntity entity) - { - await _table.ExecuteAsync(TableOperation.InsertOrReplace(entity)); - } - public async Task> GetManyAsync(string partitionKey, string rowKey, DateTime startDate, DateTime endDate, PageOptions pageOptions) { @@ -136,60 +131,28 @@ public class EventRepository : IEventRepository var end = CoreHelpers.DateTimeToTableStorageKey(endDate); var filter = MakeFilter(partitionKey, string.Format(rowKey, start), string.Format(rowKey, end)); - var query = new TableQuery().Where(filter).Take(pageOptions.PageSize); var result = new PagedResult(); - var continuationToken = DeserializeContinuationToken(pageOptions?.ContinuationToken); + var query = _tableClient.QueryAsync(filter, pageOptions.PageSize); - var queryResults = await _table.ExecuteQuerySegmentedAsync(query, continuationToken); - result.ContinuationToken = SerializeContinuationToken(queryResults.ContinuationToken); - result.Data.AddRange(queryResults.Results); + await using (var enumerator = query.AsPages(pageOptions?.ContinuationToken, + pageOptions.PageSize).GetAsyncEnumerator()) + { + await enumerator.MoveNextAsync(); + + result.ContinuationToken = enumerator.Current.ContinuationToken; + result.Data.AddRange(enumerator.Current.Values.Select(e => e.ToEventTableEntity())); + } return result; } + private async Task CreateEventAsync(EventTableEntity entity) + { + await _tableClient.UpsertEntityAsync(entity.ToAzureEvent()); + } + private string MakeFilter(string partitionKey, string rowStart, string rowEnd) { - var rowFilter = TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, $"{rowStart}`"), - TableOperators.And, - TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, $"{rowEnd}_")); - - return TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey), - TableOperators.And, - rowFilter); - } - - private string SerializeContinuationToken(TableContinuationToken token) - { - if (token == null) - { - return null; - } - - return string.Format("{0}__{1}__{2}__{3}", (int)token.TargetLocation, token.NextTableName, - token.NextPartitionKey, token.NextRowKey); - } - - private TableContinuationToken DeserializeContinuationToken(string token) - { - if (string.IsNullOrWhiteSpace(token)) - { - return null; - } - - var tokenParts = token.Split(new string[] { "__" }, StringSplitOptions.None); - if (tokenParts.Length < 4 || !Enum.TryParse(tokenParts[0], out StorageLocation tLoc)) - { - return null; - } - - return new TableContinuationToken - { - TargetLocation = tLoc, - NextTableName = string.IsNullOrWhiteSpace(tokenParts[1]) ? null : tokenParts[1], - NextPartitionKey = string.IsNullOrWhiteSpace(tokenParts[2]) ? null : tokenParts[2], - NextRowKey = string.IsNullOrWhiteSpace(tokenParts[3]) ? null : tokenParts[3] - }; + return $"PartitionKey eq '{partitionKey}' and RowKey le '{rowStart}' and RowKey ge '{rowEnd}'"; } } diff --git a/src/Core/Repositories/TableStorage/InstallationDeviceRepository.cs b/src/Core/Repositories/TableStorage/InstallationDeviceRepository.cs index 32b466d1b3..2dee07dc2b 100644 --- a/src/Core/Repositories/TableStorage/InstallationDeviceRepository.cs +++ b/src/Core/Repositories/TableStorage/InstallationDeviceRepository.cs @@ -1,13 +1,12 @@ -using System.Net; +using Azure.Data.Tables; using Bit.Core.Models.Data; using Bit.Core.Settings; -using Microsoft.Azure.Cosmos.Table; namespace Bit.Core.Repositories.TableStorage; public class InstallationDeviceRepository : IInstallationDeviceRepository { - private readonly CloudTable _table; + private readonly TableClient _tableClient; public InstallationDeviceRepository(GlobalSettings globalSettings) : this(globalSettings.Events.ConnectionString) @@ -15,14 +14,13 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository public InstallationDeviceRepository(string storageConnectionString) { - var storageAccount = CloudStorageAccount.Parse(storageConnectionString); - var tableClient = storageAccount.CreateCloudTableClient(); - _table = tableClient.GetTableReference("installationdevice"); + var tableClient = new TableServiceClient(storageConnectionString); + _tableClient = tableClient.GetTableClient("installationdevice"); } public async Task UpsertAsync(InstallationDeviceEntity entity) { - await _table.ExecuteAsync(TableOperation.InsertOrReplace(entity)); + await _tableClient.UpsertEntityAsync(entity); } public async Task UpsertManyAsync(IList entities) @@ -52,7 +50,7 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository var iterations = groupEntities.Count / 100; for (var i = 0; i <= iterations; i++) { - var batch = new TableBatchOperation(); + var batch = new List(); var batchEntities = groupEntities.Skip(i * 100).Take(100); if (!batchEntities.Any()) { @@ -61,24 +59,16 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository foreach (var entity in batchEntities) { - batch.InsertOrReplace(entity); + batch.Add(new TableTransactionAction(TableTransactionActionType.UpsertReplace, entity)); } - await _table.ExecuteBatchAsync(batch); + await _tableClient.SubmitTransactionAsync(batch); } } } public async Task DeleteAsync(InstallationDeviceEntity entity) { - try - { - entity.ETag = "*"; - await _table.ExecuteAsync(TableOperation.Delete(entity)); - } - catch (StorageException e) when (e.RequestInformation.HttpStatusCode != (int)HttpStatusCode.NotFound) - { - throw; - } + await _tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey); } }