[AC-1722] Deprecate "Edit/Delete Assigned Collections" custom permissions (#4604)

* Add SQL script to migrate custom users with specific permissions to User type

Remove 'editAssignedCollections' and 'deleteAssignedCollections' properties from Permissions in OrganizationUser table. Migrate custom users who only have these permissions to the User type.

* Add MySQL migration to migrate custom users with specific permissions to User type

* Add Postgres migration to migrate custom users with specific permissions to User type

* Add Sqlite migration to migrate custom users with specific permissions to User type

* Update AutoFixture usage in tests to resolve creating ILogger mock instances

* Update EF integration tests database contexts to use each respective Migrations assembly. Configure Sqlite instance

* Add RunMigration method to BaseEntityFrameworkRepository

* Add FinalFlexibleCollectionsDataMigrationsTests

* Improve data migration efficiency by using OPENJSON instead of multiple JSON_EXTRACT

* Add batching to the sql data migrations

* Update DbMigrator to run a specific script based on its name

* Update DatabaseDataAttribute to be able to test a specific migration

* Add reference to the migration projects to Infrastructure.IntegrationTest

* Add integration test to test the migration FinalFlexibleCollectionsDataMigrations

* Remove EFIntegration tests and remove RunMigration method from BaseEntityFrameworkRepository

* Add IMigrationTesterService and implementations for SQL and EF migrations

* Add FinalFlexibleCollectionsDataMigrationsTests and remove test from OrganizationUserRepositoryTests

* Update sql data migration script based on performance feedback

* Bump date on EF migration scripts

* Add xmldoc comments to IMigrationTesterService and each implementation

* Bump up the date on the EF migration scripts

* Bump up dates on EF migrations

* Added tests to assert no unwanted changes are made to the permissions json. Refactor tests.

* Revert changes made to DbMigrator and refactor SqlMigrationTesterService to not use it.

* Add method description

* Fix test to assert no changes are made to custom user

* Remove unnecessary COALESCE and SELECT CASE

* Unident lines on SQL script

* Update DatabaseDataAttribute MigrationName property to be nullable

* Fix null reference checks

* Remove unnecessary COALESCE from Postgres script

* Bump dates on migration scripts

* Bump up dates on EF migrations

* Add migration tests for handling null

* Add test for non json values

* Fix test

* Remove migrations

* Recreate EF migrations

* Update Postgres data migration script to check for valid JSON in Permissions column

---------

Co-authored-by: Thomas Rittson <31796059+eliykat@users.noreply.github.com>
Co-authored-by: Thomas Rittson <trittson@bitwarden.com>

test/Common/AutoFixture/Attributes/CustomAutoDataAttribute.cs

@@ -11,7 +11,7 @@ public class CustomAutoDataAttribute : AutoDataAttribute
 
     public CustomAutoDataAttribute(params ICustomization[] customizations) : base(() =>
     {
-        var fixture = new Fixture();
+        var fixture = new Fixture().WithAutoNSubstitutions();
         foreach (var customization in customizations)
         {
             fixture.Customize(customization);

test/Infrastructure.EFIntegration.Test/AutoFixture/EntityFrameworkRepositoryFixtures.cs

@@ -12,6 +12,7 @@ using Bit.Infrastructure.EntityFramework.Repositories;
 using Bit.Infrastructure.EntityFramework.Vault.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using NSubstitute;
 
 namespace Bit.Infrastructure.EFIntegration.Test.AutoFixture;
@@ -91,6 +92,8 @@ public class EfRepositoryListBuilder<T> : ISpecimenBuilder where T : BaseEntityF
                 })
             .CreateMapper()));
 
+            fixture.Customize<ILogger<T>>(x => x.FromFactory(() => Substitute.For<ILogger<T>>()));
+
             var repo = fixture.Create<T>();
             list.Add(repo);
         }

test/Infrastructure.EFIntegration.Test/Helpers/DatabaseOptionsFactory.cs

@@ -37,7 +37,10 @@ public static class DatabaseOptionsFactory
         {
             AppContext.SetSwitch("Npgsql.EnableLegacyTimestampBehavior", true);
             Options.Add(new DbContextOptionsBuilder<DatabaseContext>()
-                .UseNpgsql(globalSettings.PostgreSql.ConnectionString)
+                .UseNpgsql(globalSettings.PostgreSql.ConnectionString, npgsqlDbContextOptionsBuilder =>
+                {
+                    npgsqlDbContextOptionsBuilder.MigrationsAssembly(nameof(PostgresMigrations));
+                })
                 .UseApplicationServiceProvider(services)
                 .Options);
         }
@@ -45,7 +48,21 @@ public static class DatabaseOptionsFactory
         {
             var mySqlConnectionString = globalSettings.MySql.ConnectionString;
             Options.Add(new DbContextOptionsBuilder<DatabaseContext>()
-                .UseMySql(mySqlConnectionString, ServerVersion.AutoDetect(mySqlConnectionString))
+                .UseMySql(mySqlConnectionString, ServerVersion.AutoDetect(mySqlConnectionString), mySqlDbContextOptionsBuilder =>
+                {
+                    mySqlDbContextOptionsBuilder.MigrationsAssembly(nameof(MySqlMigrations));
+                })
+                .UseApplicationServiceProvider(services)
+                .Options);
+        }
+        if (!string.IsNullOrWhiteSpace(GlobalSettingsFactory.GlobalSettings.Sqlite?.ConnectionString))
+        {
+            var sqliteConnectionString = globalSettings.Sqlite.ConnectionString;
+            Options.Add(new DbContextOptionsBuilder<DatabaseContext>()
+                .UseSqlite(sqliteConnectionString, mySqlDbContextOptionsBuilder =>
+                {
+                    mySqlDbContextOptionsBuilder.MigrationsAssembly(nameof(SqliteMigrations));
+                })
                 .UseApplicationServiceProvider(services)
                 .Options);
         }

test/Infrastructure.EFIntegration.Test/Infrastructure.EFIntegration.Test.csproj

@@ -21,6 +21,9 @@
     <ItemGroup>
       <ProjectReference Include="..\..\src\Infrastructure.Dapper\Infrastructure.Dapper.csproj" />
       <ProjectReference Include="..\..\src\Infrastructure.EntityFramework\Infrastructure.EntityFramework.csproj" />
+      <ProjectReference Include="..\..\util\MySqlMigrations\MySqlMigrations.csproj" />
+      <ProjectReference Include="..\..\util\PostgresMigrations\PostgresMigrations.csproj" />
+      <ProjectReference Include="..\..\util\SqliteMigrations\SqliteMigrations.csproj" />
       <ProjectReference Include="..\Common\Common.csproj" />
       <ProjectReference Include="..\Core.Test\Core.Test.csproj" />
     </ItemGroup>

test/Infrastructure.IntegrationTest/AdminConsole/Migrations/FinalFlexibleCollectionsDataMigrationsTests.cs

@@ -0,0 +1,335 @@
+using System.Text.Json;
+using Bit.Core.AdminConsole.Entities;
+using Bit.Core.Entities;
+using Bit.Core.Enums;
+using Bit.Core.Models.Data;
+using Bit.Core.Repositories;
+using Bit.Core.Utilities;
+using Bit.Infrastructure.IntegrationTest.Services;
+using Newtonsoft.Json.Linq;
+using Xunit;
+
+namespace Bit.Infrastructure.IntegrationTest.AdminConsole.Migrations;
+
+public class FinalFlexibleCollectionsDataMigrationsTests
+{
+    private const string _migrationName = "FinalFlexibleCollectionsDataMigrations";
+
+    [DatabaseTheory, DatabaseData(MigrationName = _migrationName)]
+    public async Task RunMigration_WithEditAssignedCollections_WithCustomUserType_MigratesToUserNullPermissions(
+        IUserRepository userRepository,
+        IOrganizationRepository organizationRepository,
+        IOrganizationUserRepository organizationUserRepository,
+        IMigrationTesterService migrationTester)
+    {
+        // Setup data
+        var orgUser = await SetupData(
+            userRepository, organizationRepository, organizationUserRepository,
+            OrganizationUserType.Custom, editAssignedCollections: true, deleteAssignedCollections: false);
+
+        // Run data migration
+        migrationTester.ApplyMigration();
+
+        // Assert that the user was migrated to a User type with null permissions
+        var migratedOrgUser = await organizationUserRepository.GetByIdAsync(orgUser.Id);
+        Assert.NotNull(migratedOrgUser);
+        Assert.Equal(orgUser.Id, migratedOrgUser.Id);
+        Assert.Equal(OrganizationUserType.User, migratedOrgUser.Type);
+        Assert.Null(migratedOrgUser.Permissions);
+    }
+
+    [DatabaseTheory, DatabaseData(MigrationName = _migrationName)]
+    public async Task RunMigration_WithDeleteAssignedCollections_WithCustomUserType_MigratesToUserNullPermissions(
+        IUserRepository userRepository,
+        IOrganizationRepository organizationRepository,
+        IOrganizationUserRepository organizationUserRepository,
+        IMigrationTesterService migrationTester)
+    {
+        // Setup data
+        var orgUser = await SetupData(
+            userRepository, organizationRepository, organizationUserRepository,
+            OrganizationUserType.Custom, editAssignedCollections: false, deleteAssignedCollections: true);
+
+        // Run data migration
+        migrationTester.ApplyMigration();
+
+        // Assert that the user was migrated to a User type with null permissions
+        var migratedOrgUser = await organizationUserRepository.GetByIdAsync(orgUser.Id);
+        Assert.NotNull(migratedOrgUser);
+        Assert.Equal(orgUser.Id, migratedOrgUser.Id);
+        Assert.Equal(OrganizationUserType.User, migratedOrgUser.Type);
+        Assert.Null(migratedOrgUser.Permissions);
+    }
+
+    [DatabaseTheory, DatabaseData(MigrationName = _migrationName)]
+    public async Task RunMigration_WithEditAndDeleteAssignedCollections_WithCustomUserType_MigratesToUserNullPermissions(
+        IUserRepository userRepository,
+        IOrganizationRepository organizationRepository,
+        IOrganizationUserRepository organizationUserRepository,
+        IMigrationTesterService migrationTester)
+    {
+        // Setup data
+        var orgUser = await SetupData(
+            userRepository, organizationRepository, organizationUserRepository,
+            OrganizationUserType.Custom, editAssignedCollections: true, deleteAssignedCollections: true);
+
+        // Run data migration
+        migrationTester.ApplyMigration();
+
+        // Assert that the user was migrated to a User type with null permissions
+        var migratedOrgUser = await organizationUserRepository.GetByIdAsync(orgUser.Id);
+        Assert.NotNull(migratedOrgUser);
+        Assert.Equal(orgUser.Id, migratedOrgUser.Id);
+        Assert.Equal(OrganizationUserType.User, migratedOrgUser.Type);
+        Assert.Null(migratedOrgUser.Permissions);
+    }
+
+    [DatabaseTheory, DatabaseData(MigrationName = _migrationName)]
+    public async Task RunMigration_WithoutAssignedCollectionsPermissions_WithCustomUserType_RemovesAssignedCollectionsPermissions(
+        IUserRepository userRepository,
+        IOrganizationRepository organizationRepository,
+        IOrganizationUserRepository organizationUserRepository,
+        IMigrationTesterService migrationTester)
+    {
+        // Setup data
+        var orgUser = await SetupData(
+            userRepository, organizationRepository, organizationUserRepository, OrganizationUserType.Custom,
+            editAssignedCollections: false, deleteAssignedCollections: false, accessEventLogs: true);
+
+        // Run data migration
+        migrationTester.ApplyMigration();
+
+        // Assert that the user kept the accessEventLogs permission and lost the editAssignedCollections and deleteAssignedCollections permissions
+        var migratedOrgUser = await organizationUserRepository.GetByIdAsync(orgUser.Id);
+        Assert.NotNull(migratedOrgUser);
+        Assert.Equal(orgUser.Id, migratedOrgUser.Id);
+        Assert.Equal(OrganizationUserType.Custom, migratedOrgUser.Type);
+        Assert.NotEqual(orgUser.Permissions, migratedOrgUser.Permissions);
+        Assert.NotNull(migratedOrgUser.Permissions);
+        Assert.Contains("accessEventLogs", orgUser.Permissions);
+        Assert.Contains("editAssignedCollections", orgUser.Permissions);
+        Assert.Contains("deleteAssignedCollections", orgUser.Permissions);
+
+        Assert.Contains("accessEventLogs", migratedOrgUser.Permissions);
+        var migratedOrgUserPermissions = migratedOrgUser.GetPermissions();
+        Assert.NotNull(migratedOrgUserPermissions);
+        Assert.True(migratedOrgUserPermissions.AccessEventLogs);
+        Assert.DoesNotContain("editAssignedCollections", migratedOrgUser.Permissions);
+        Assert.DoesNotContain("deleteAssignedCollections", migratedOrgUser.Permissions);
+    }
+
+    [DatabaseTheory, DatabaseData(MigrationName = _migrationName)]
+    public async Task RunMigration_WithAdminUserType_RemovesAssignedCollectionsPermissions(
+        IUserRepository userRepository,
+        IOrganizationRepository organizationRepository,
+        IOrganizationUserRepository organizationUserRepository,
+        IMigrationTesterService migrationTester)
+    {
+        // Setup data
+        var orgUser = await SetupData(
+            userRepository, organizationRepository, organizationUserRepository, OrganizationUserType.Admin,
+            editAssignedCollections: false, deleteAssignedCollections: false, accessEventLogs: true);
+
+        // Run data migration
+        migrationTester.ApplyMigration();
+
+        // Assert that the user kept the Admin type and lost the editAssignedCollections and deleteAssignedCollections
+        // permissions but kept the accessEventLogs permission
+        var migratedOrgUser = await organizationUserRepository.GetByIdAsync(orgUser.Id);
+        Assert.NotNull(migratedOrgUser);
+        Assert.Equal(orgUser.Id, migratedOrgUser.Id);
+        Assert.Equal(OrganizationUserType.Admin, migratedOrgUser.Type);
+        Assert.NotEqual(orgUser.Permissions, migratedOrgUser.Permissions);
+        Assert.NotNull(migratedOrgUser.Permissions);
+        Assert.Contains("accessEventLogs", orgUser.Permissions);
+        Assert.Contains("editAssignedCollections", orgUser.Permissions);
+        Assert.Contains("deleteAssignedCollections", orgUser.Permissions);
+
+        Assert.Contains("accessEventLogs", migratedOrgUser.Permissions);
+        Assert.True(migratedOrgUser.GetPermissions().AccessEventLogs);
+        Assert.DoesNotContain("editAssignedCollections", migratedOrgUser.Permissions);
+        Assert.DoesNotContain("deleteAssignedCollections", migratedOrgUser.Permissions);
+    }
+
+    [DatabaseTheory, DatabaseData(MigrationName = _migrationName)]
+    public async Task RunMigration_WithoutAssignedCollectionsPermissions_DoesNothing(
+        IUserRepository userRepository,
+        IOrganizationRepository organizationRepository,
+        IOrganizationUserRepository organizationUserRepository,
+        IMigrationTesterService migrationTester)
+    {
+        // Setup data
+        var orgUser = await SetupData(
+            userRepository, organizationRepository, organizationUserRepository, OrganizationUserType.Custom,
+            editAssignedCollections: false, deleteAssignedCollections: false, accessEventLogs: false);
+        // Remove the editAssignedCollections and deleteAssignedCollections permissions
+        orgUser.Permissions = JsonSerializer.Serialize(new
+        {
+            AccessEventLogs = false,
+            AccessImportExport = false,
+            AccessReports = false,
+            CreateNewCollections = false,
+            EditAnyCollection = false,
+            DeleteAnyCollection = false,
+            ManageGroups = false,
+            ManagePolicies = false,
+            ManageSso = false,
+            ManageUsers = false,
+            ManageResetPassword = false,
+            ManageScim = false
+        }, JsonHelpers.CamelCase);
+        await organizationUserRepository.ReplaceAsync(orgUser);
+
+        // Run data migration
+        migrationTester.ApplyMigration();
+
+        // Assert that the user remained unchanged
+        var migratedOrgUser = await organizationUserRepository.GetByIdAsync(orgUser.Id);
+        Assert.NotNull(migratedOrgUser);
+        Assert.Equal(orgUser.Id, migratedOrgUser.Id);
+        Assert.Equal(OrganizationUserType.Custom, orgUser.Type);
+        Assert.Equal(OrganizationUserType.Custom, migratedOrgUser.Type);
+        Assert.NotNull(migratedOrgUser.Permissions);
+        // Assert that the permissions remain unchanged by comparing JSON data, ignoring the order of properties
+        Assert.True(JToken.DeepEquals(JObject.Parse(orgUser.Permissions), JObject.Parse(migratedOrgUser.Permissions)));
+    }
+
+    [DatabaseTheory, DatabaseData(MigrationName = _migrationName)]
+    public async Task RunMigration_HandlesNull(
+        IUserRepository userRepository,
+        IOrganizationRepository organizationRepository,
+        IOrganizationUserRepository organizationUserRepository,
+        IMigrationTesterService migrationTester)
+    {
+        // Setup data
+        var orgUser = await SetupData(
+            userRepository, organizationRepository, organizationUserRepository, OrganizationUserType.Custom,
+            editAssignedCollections: false, deleteAssignedCollections: false, accessEventLogs: false);
+
+        orgUser.Permissions = null;
+        await organizationUserRepository.ReplaceAsync(orgUser);
+
+        // Run data migration
+        migrationTester.ApplyMigration();
+
+        // Assert no changes
+        var migratedOrgUser = await organizationUserRepository.GetByIdAsync(orgUser.Id);
+        Assert.NotNull(migratedOrgUser);
+        Assert.Equal(orgUser.Id, migratedOrgUser.Id);
+        Assert.Equal(orgUser.Type, migratedOrgUser.Type);
+        Assert.Null(migratedOrgUser.Permissions);
+    }
+
+    [DatabaseTheory, DatabaseData(MigrationName = _migrationName)]
+    public async Task RunMigration_HandlesNullString(
+        IUserRepository userRepository,
+        IOrganizationRepository organizationRepository,
+        IOrganizationUserRepository organizationUserRepository,
+        IMigrationTesterService migrationTester)
+    {
+        // Setup data
+        var orgUser = await SetupData(
+            userRepository, organizationRepository, organizationUserRepository, OrganizationUserType.Custom,
+            editAssignedCollections: false, deleteAssignedCollections: false, accessEventLogs: false);
+
+        // We haven't tracked down the source of this yet but it does occur in our cloud database
+        orgUser.Permissions = "NULL";
+        await organizationUserRepository.ReplaceAsync(orgUser);
+
+        // Run data migration
+        migrationTester.ApplyMigration();
+
+        // Assert no changes
+        var migratedOrgUser = await organizationUserRepository.GetByIdAsync(orgUser.Id);
+        Assert.NotNull(migratedOrgUser);
+        Assert.Equal(orgUser.Id, migratedOrgUser.Id);
+        Assert.Equal(orgUser.Type, migratedOrgUser.Type);
+        Assert.Equal("NULL", migratedOrgUser.Permissions);
+    }
+
+    [DatabaseTheory, DatabaseData(MigrationName = _migrationName)]
+    public async Task RunMigration_HandlesNonJsonValues(
+        IUserRepository userRepository,
+        IOrganizationRepository organizationRepository,
+        IOrganizationUserRepository organizationUserRepository,
+        IMigrationTesterService migrationTester)
+    {
+        // Setup data
+        var orgUser = await SetupData(
+            userRepository, organizationRepository, organizationUserRepository, OrganizationUserType.Custom,
+            editAssignedCollections: false, deleteAssignedCollections: false, accessEventLogs: false);
+
+        orgUser.Permissions = "asdfasdfasfd";
+        await organizationUserRepository.ReplaceAsync(orgUser);
+
+        // Run data migration
+        migrationTester.ApplyMigration();
+
+        // Assert no changes
+        var migratedOrgUser = await organizationUserRepository.GetByIdAsync(orgUser.Id);
+        Assert.NotNull(migratedOrgUser);
+        Assert.Equal(orgUser.Id, migratedOrgUser.Id);
+        Assert.Equal(orgUser.Type, migratedOrgUser.Type);
+        Assert.Equal("asdfasdfasfd", migratedOrgUser.Permissions);
+    }
+
+    private async Task<OrganizationUser> SetupData(
+        IUserRepository userRepository,
+        IOrganizationRepository organizationRepository,
+        IOrganizationUserRepository organizationUserRepository,
+        OrganizationUserType organizationUserType,
+        bool editAssignedCollections,
+        bool deleteAssignedCollections,
+        bool accessEventLogs = false)
+    {
+        var permissions = new Permissions
+        {
+            AccessEventLogs = accessEventLogs,
+            AccessImportExport = false,
+            AccessReports = false,
+            CreateNewCollections = false,
+            EditAnyCollection = false,
+            DeleteAnyCollection = false,
+            EditAssignedCollections = editAssignedCollections,
+            DeleteAssignedCollections = deleteAssignedCollections,
+            ManageGroups = false,
+            ManagePolicies = false,
+            ManageSso = false,
+            ManageUsers = false,
+            ManageResetPassword = false,
+            ManageScim = false
+        };
+
+        var user = await userRepository.CreateAsync(new User
+        {
+            Name = "Test User 1",
+            Email = $"test+{Guid.NewGuid()}@example.com",
+            ApiKey = "TEST",
+            SecurityStamp = "stamp",
+            Kdf = KdfType.PBKDF2_SHA256,
+            KdfIterations = 1,
+            KdfMemory = 2,
+            KdfParallelism = 3
+        });
+
+        var organization = await organizationRepository.CreateAsync(new Organization
+        {
+            Name = "Test Org",
+            BillingEmail = user.Email, // TODO: EF does not enforce this being NOT NULl
+            Plan = "Test", // TODO: EF does not enforce this being NOT NULl
+            PrivateKey = "privatekey",
+        });
+
+        var orgUser = await organizationUserRepository.CreateAsync(new OrganizationUser
+        {
+            OrganizationId = organization.Id,
+            UserId = user.Id,
+            Status = OrganizationUserStatusType.Confirmed,
+            ResetPasswordKey = "resetpasswordkey1",
+            Type = organizationUserType,
+            Permissions = JsonSerializer.Serialize(permissions, JsonHelpers.CamelCase)
+        });
+
+        return orgUser;
+    }
+}

test/Infrastructure.IntegrationTest/DatabaseDataAttribute.cs

@@ -3,6 +3,8 @@ using Bit.Core.Enums;
 using Bit.Core.Settings;
 using Bit.Infrastructure.Dapper;
 using Bit.Infrastructure.EntityFramework;
+using Bit.Infrastructure.EntityFramework.Repositories;
+using Bit.Infrastructure.IntegrationTest.Services;
 using Microsoft.Extensions.Caching.Distributed;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -16,6 +18,7 @@ public class DatabaseDataAttribute : DataAttribute
 {
     public bool SelfHosted { get; set; }
     public bool UseFakeTimeProvider { get; set; }
+    public string? MigrationName { get; set; }
 
     public override IEnumerable<object[]> GetData(MethodInfo testMethod)
     {
@@ -74,6 +77,12 @@ public class DatabaseDataAttribute : DataAttribute
                     o.SchemaName = "dbo";
                     o.TableName = "Cache";
                 });
+
+                if (!string.IsNullOrEmpty(MigrationName))
+                {
+                    AddSqlMigrationTester(dapperSqlServerCollection, database.ConnectionString, MigrationName);
+                }
+
                 yield return dapperSqlServerCollection.BuildServiceProvider();
             }
             else
@@ -84,6 +93,12 @@ public class DatabaseDataAttribute : DataAttribute
                 efCollection.AddPasswordManagerEFRepositories(SelfHosted);
                 efCollection.AddSingleton(database);
                 efCollection.AddSingleton<IDistributedCache, EntityFrameworkCache>();
+
+                if (!string.IsNullOrEmpty(MigrationName))
+                {
+                    AddEfMigrationTester(efCollection, database.Type, MigrationName);
+                }
+
                 yield return efCollection.BuildServiceProvider();
             }
         }
@@ -99,4 +114,18 @@ public class DatabaseDataAttribute : DataAttribute
             services.AddSingleton<TimeProvider, FakeTimeProvider>();
         }
     }
+
+    private void AddSqlMigrationTester(IServiceCollection services, string connectionString, string migrationName)
+    {
+        services.AddSingleton<IMigrationTesterService, SqlMigrationTesterService>(sp => new SqlMigrationTesterService(connectionString, migrationName));
+    }
+
+    private void AddEfMigrationTester(IServiceCollection services, SupportedDatabaseProviders databaseType, string migrationName)
+    {
+        services.AddSingleton<IMigrationTesterService, EfMigrationTesterService>(sp =>
+        {
+            var dbContext = sp.GetRequiredService<DatabaseContext>();
+            return new EfMigrationTesterService(dbContext, databaseType, migrationName);
+        });
+    }
 }

test/Infrastructure.IntegrationTest/Infrastructure.IntegrationTest.csproj

@@ -27,6 +27,10 @@
   <ItemGroup>
     <ProjectReference Include="..\..\src\Infrastructure.Dapper\Infrastructure.Dapper.csproj" />
     <ProjectReference Include="..\..\src\Infrastructure.EntityFramework\Infrastructure.EntityFramework.csproj" />
+    <ProjectReference Include="..\..\util\Migrator\Migrator.csproj" />
+    <ProjectReference Include="..\..\util\MySqlMigrations\MySqlMigrations.csproj" />
+    <ProjectReference Include="..\..\util\PostgresMigrations\PostgresMigrations.csproj" />
+    <ProjectReference Include="..\..\util\SqliteMigrations\SqliteMigrations.csproj" />
   </ItemGroup>
 
 </Project>

test/Infrastructure.IntegrationTest/Services/IMigrationTesterService.cs

@@ -0,0 +1,19 @@
+namespace Bit.Infrastructure.IntegrationTest.Services;
+
+/// <summary>
+/// Defines the contract for applying a specific database migration across different database providers.
+/// Implementations of this interface are responsible for migration execution logic,
+/// and handling migration history to ensure that migrations can be tested independently and reliably.
+/// </summary>
+/// <remarks>
+/// Each implementation should receive the migration name as a parameter in the constructor
+/// to specify which migration is to be applied.
+/// </remarks>
+public interface IMigrationTesterService
+{
+    /// <summary>
+    /// Applies the specified database migration.
+    /// This may involve managing migration history and retry logic, depending on the implementation.
+    /// </summary>
+    void ApplyMigration();
+}

test/Infrastructure.IntegrationTest/Services/Implementations/EfMigrationTesterService.cs

@@ -0,0 +1,69 @@
+using System.Data;
+using Bit.Core.Enums;
+using Bit.Infrastructure.EntityFramework.Repositories;
+using Microsoft.Data.Sqlite;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using MySqlConnector;
+using Npgsql;
+
+namespace Bit.Infrastructure.IntegrationTest.Services;
+
+/// <summary>
+/// An implementation of <see cref="IMigrationTesterService"/> for testing Entity Framework migrations.
+/// This service applies a specific migration and manages the migration history
+/// to ensure that the migration is tested in isolation. It supports MySQL, Postgres, and SQLite.
+/// </summary>
+public class EfMigrationTesterService : IMigrationTesterService
+{
+    private readonly DatabaseContext _databaseContext;
+    private readonly SupportedDatabaseProviders _databaseType;
+    private readonly string _migrationName;
+
+    public EfMigrationTesterService(
+        DatabaseContext databaseContext,
+        SupportedDatabaseProviders databaseType,
+        string migrationName)
+    {
+        _databaseContext = databaseContext;
+        _databaseType = databaseType;
+        _migrationName = migrationName;
+    }
+
+    public void ApplyMigration()
+    {
+        // Delete the migration history to ensure the migration is applied
+        DeleteMigrationHistory();
+
+        var migrator = _databaseContext.GetService<IMigrator>();
+        migrator.Migrate(_migrationName);
+    }
+
+    /// <summary>
+    /// Deletes the migration history for the specified migration name.
+    /// </summary>
+    private void DeleteMigrationHistory()
+    {
+        var deleteCommand = "DELETE FROM __EFMigrationsHistory WHERE MigrationId LIKE @migrationName";
+        IDbDataParameter? parameter;
+
+        switch (_databaseType)
+        {
+            case SupportedDatabaseProviders.MySql:
+                parameter = new MySqlParameter("@migrationName", "%" + _migrationName);
+                break;
+            case SupportedDatabaseProviders.Postgres:
+                deleteCommand = "DELETE FROM \"__EFMigrationsHistory\" WHERE \"MigrationId\" LIKE @migrationName";
+                parameter = new NpgsqlParameter("@migrationName", "%" + _migrationName);
+                break;
+            case SupportedDatabaseProviders.Sqlite:
+                parameter = new SqliteParameter("@migrationName", "%" + _migrationName);
+                break;
+            default:
+                throw new InvalidOperationException($"Unsupported database type: {_databaseType}");
+        }
+
+        _databaseContext.Database.ExecuteSqlRaw(deleteCommand, parameter);
+    }
+}

test/Infrastructure.IntegrationTest/Services/Implementations/SqlMigrationTesterService.cs

@@ -0,0 +1,60 @@
+using Bit.Migrator;
+using Microsoft.Data.SqlClient;
+
+namespace Bit.Infrastructure.IntegrationTest.Services;
+
+/// <summary>
+/// An implementation of <see cref="IMigrationTesterService"/> for testing SQL Server migrations.
+/// This service applies a specified SQL migration script to a SQL Server database.
+/// </summary>
+public class SqlMigrationTesterService : IMigrationTesterService
+{
+    private readonly string _connectionString;
+    private readonly string _migrationName;
+
+    public SqlMigrationTesterService(string connectionString, string migrationName)
+    {
+        _connectionString = connectionString;
+        _migrationName = migrationName;
+    }
+
+    public void ApplyMigration()
+    {
+        var script = GetMigrationScript(_migrationName);
+
+        using var connection = new SqlConnection(_connectionString);
+        connection.Open();
+
+        using var transaction = connection.BeginTransaction();
+        try
+        {
+            using (var command = new SqlCommand(script, connection, transaction))
+            {
+                command.ExecuteNonQuery();
+            }
+
+            transaction.Commit();
+        }
+        catch
+        {
+            transaction.Rollback();
+            throw;
+        }
+    }
+
+    private string GetMigrationScript(string scriptName)
+    {
+        var assembly = typeof(DbMigrator).Assembly; ;
+        var resourceName = assembly.GetManifestResourceNames()
+            .FirstOrDefault(r => r.EndsWith($"{scriptName}.sql"));
+
+        if (resourceName == null)
+        {
+            throw new FileNotFoundException($"SQL migration script file for '{scriptName}' was not found.");
+        }
+
+        using var stream = assembly.GetManifestResourceStream(resourceName);
+        using var reader = new StreamReader(stream!);
+        return reader.ReadToEnd();
+    }
+}

util/Migrator/DbScripts/2024-08-26_00_FinalFlexibleCollectionsDataMigrations.sql

@@ -0,0 +1,118 @@
+DECLARE @BatchSize INT = 2000;
+DECLARE @RowsAffected INT;
+
+-- Migrate Custom users who only have 'editAssignedCollections' and/or 'deleteAssignedCollections'
+-- custom permissions to the User type.
+WHILE 1 = 1
+BEGIN
+    UPDATE TOP (@BatchSize) [dbo].[OrganizationUser]
+    SET
+        [Type] = 2,
+        [Permissions] = NULL
+    WHERE
+        [Type] = 4
+        AND ISJSON([Permissions]) = 1
+        AND EXISTS (
+            SELECT 1
+            FROM OPENJSON([Permissions])
+            WITH (
+                editAssignedCollections bit '$.editAssignedCollections',
+                deleteAssignedCollections bit '$.deleteAssignedCollections',
+                accessEventLogs bit '$.accessEventLogs',
+                accessImportExport bit '$.accessImportExport',
+                accessReports bit '$.accessReports',
+                createNewCollections bit '$.createNewCollections',
+                editAnyCollection bit '$.editAnyCollection',
+                deleteAnyCollection bit '$.deleteAnyCollection',
+                manageGroups bit '$.manageGroups',
+                managePolicies bit '$.managePolicies',
+                manageSso bit '$.manageSso',
+                manageUsers bit '$.manageUsers',
+                manageResetPassword bit '$.manageResetPassword',
+                manageScim bit '$.manageScim'
+            ) AS PermissionsJson
+            WHERE
+                (PermissionsJson.editAssignedCollections = 1 OR PermissionsJson.deleteAssignedCollections = 1)
+                AND PermissionsJson.accessEventLogs = 0
+                AND PermissionsJson.accessImportExport = 0
+                AND PermissionsJson.accessReports = 0
+                AND PermissionsJson.createNewCollections = 0
+                AND PermissionsJson.editAnyCollection = 0
+                AND PermissionsJson.deleteAnyCollection = 0
+                AND PermissionsJson.manageGroups = 0
+                AND PermissionsJson.managePolicies = 0
+                AND PermissionsJson.manageSso = 0
+                AND PermissionsJson.manageUsers = 0
+                AND PermissionsJson.manageResetPassword = 0
+                AND PermissionsJson.manageScim = 0
+        );
+
+    SET @RowsAffected = @@ROWCOUNT;
+
+    IF @RowsAffected = 0
+        BREAK;
+END
+
+-- Remove 'editAssignedCollections' and 'deleteAssignedCollections' properties from Permissions
+-- Step 1: Create a temporary table to store the IDs and parsed JSON values
+CREATE TABLE #TempIds (
+    TempId INT IDENTITY(1,1) PRIMARY KEY,
+    OrganizationUserId UNIQUEIDENTIFIER,
+    editAssignedCollections BIT,
+    deleteAssignedCollections BIT
+);
+
+-- Step 2: Populate the temporary table with the IDs and parsed JSON values
+INSERT INTO #TempIds (OrganizationUserId, editAssignedCollections, deleteAssignedCollections)
+SELECT
+    Id,
+    CAST(JSON_VALUE([Permissions], '$.editAssignedCollections') AS BIT) AS editAssignedCollections,
+    CAST(JSON_VALUE([Permissions], '$.deleteAssignedCollections') AS BIT) AS deleteAssignedCollections
+FROM [dbo].[OrganizationUser]
+WHERE
+    ISJSON([Permissions]) = 1
+    AND (
+        JSON_VALUE([Permissions], '$.editAssignedCollections') IS NOT NULL
+        OR JSON_VALUE([Permissions], '$.deleteAssignedCollections') IS NOT NULL
+    );
+
+DECLARE @MaxTempId INT;
+DECLARE @CurrentBatchStart INT = 1;
+
+-- Get the maximum TempId
+SELECT @MaxTempId = MAX(TempId) FROM #TempIds;
+
+-- Step 3: Loop through the IDs in batches
+WHILE @CurrentBatchStart <= @MaxTempId
+BEGIN
+    UPDATE tu
+    SET
+        [Permissions] =
+            JSON_MODIFY(
+                JSON_MODIFY(
+                    [Permissions],
+                    '$.editAssignedCollections',
+                    NULL
+                ),
+                '$.deleteAssignedCollections',
+                NULL
+            )
+    FROM [dbo].[OrganizationUser] tu
+    INNER JOIN #TempIds ti ON tu.Id = ti.OrganizationUserId
+    WHERE
+        ti.TempId BETWEEN @CurrentBatchStart AND @CurrentBatchStart + @BatchSize - 1
+        AND (
+            ti.editAssignedCollections IS NOT NULL
+            OR ti.deleteAssignedCollections IS NOT NULL
+        );
+
+    SET @RowsAffected = @@ROWCOUNT;
+
+    IF @RowsAffected = 0
+        BREAK;
+
+    SET @CurrentBatchStart = @CurrentBatchStart + @BatchSize;
+END
+
+-- Clean up the temporary table
+DROP TABLE #TempIds;

util/MsSqlMigratorUtility/Program.cs

@@ -27,7 +27,7 @@ internal class Program
         bool success;
         if (!string.IsNullOrWhiteSpace(folderName))
         {
-            success = migrator.MigrateMsSqlDatabaseWithRetries(true, repeatable, folderName, dryRun);
+            success = migrator.MigrateMsSqlDatabaseWithRetries(true, repeatable, folderName, dryRun: dryRun);
         }
         else
         {

util/MySqlMigrations/HelperScripts/2024-08-26_00_FinalFlexibleCollectionsDataMigrations.sql

@@ -0,0 +1,41 @@
+-- Migrate Custom users who only have 'editAssignedCollections' and/or 'deleteAssignedCollections' custom permissions to the User type.
+UPDATE `OrganizationUser`
+SET
+    `Type` = 2,
+    `Permissions` = NULL
+WHERE
+    `Type` = 4
+    AND JSON_VALID(`Permissions`) = 1
+    AND (
+        JSON_VALUE(`Permissions`, '$.editAssignedCollections') = 'true'
+        OR JSON_VALUE(`Permissions`, '$.deleteAssignedCollections') = 'true'
+    )
+    AND JSON_VALUE(`Permissions`, '$.accessEventLogs') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.accessImportExport') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.accessReports') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.createNewCollections') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.editAnyCollection') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.deleteAnyCollection') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.manageGroups') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.managePolicies') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.manageSso') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.manageUsers') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.manageResetPassword') = 'false'
+    AND JSON_VALUE(`Permissions`, '$.manageScim') = 'false';
+
+-- Remove 'editAssignedCollections' and 'deleteAssignedCollections' properties from Permissions
+UPDATE `OrganizationUser`
+SET
+    `Permissions` = JSON_REMOVE(
+            JSON_REMOVE(
+                `Permissions`,
+                 '$.editAssignedCollections'
+            ),
+            '$.deleteAssignedCollections'
+        )
+WHERE
+    JSON_VALID(`Permissions`) = 1
+    AND (
+        JSON_VALUE(`Permissions`, '$.editAssignedCollections') IS NOT NULL
+        OR JSON_VALUE(`Permissions`, '$.deleteAssignedCollections') IS NOT NULL
+    );

util/MySqlMigrations/Migrations/20240828101433_FinalFlexibleCollectionsDataMigrations.cs

@@ -0,0 +1,21 @@
+using Bit.Core.Utilities;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Bit.MySqlMigrations.Migrations;
+
+public partial class FinalFlexibleCollectionsDataMigrations : Migration
+{
+    private const string _finalFlexibleCollectionsDataMigrationsScript = "MySqlMigrations.HelperScripts.2024-08-26_00_FinalFlexibleCollectionsDataMigrations.sql";
+
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql(CoreHelpers.GetEmbeddedResourceContentsAsync(_finalFlexibleCollectionsDataMigrationsScript));
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        throw new Exception("Irreversible migration");
+    }
+}

util/MySqlMigrations/MySqlMigrations.csproj

@@ -30,5 +30,6 @@
     <EmbeddedResource Include="HelperScripts\2022-03-01_00_Up_MigrateOrganizationApiKeys.sql" />
     <EmbeddedResource Include="HelperScripts\2022-03-01_00_Down_MigrateOrganizationApiKeys.sql" />
     <EmbeddedResource Include="HelperScripts\2024-04-25_00_EnableOrgsCollectionEnhancements.sql" />
+    <EmbeddedResource Include="HelperScripts\2024-08-26_00_FinalFlexibleCollectionsDataMigrations.sql" />
   </ItemGroup>
 </Project>

util/PostgresMigrations/HelperScripts/2024-08-26_00_FinalFlexibleCollectionsDataMigrations.psql

@@ -0,0 +1,35 @@
+-- Migrate Custom users who only have 'editAssignedCollections' and/or 'deleteAssignedCollections' custom permissions to the User type.
+UPDATE "OrganizationUser"
+SET
+    "Type" = 2,
+    "Permissions" = NULL
+WHERE
+    "Type" = 4
+    AND "Permissions" IS NOT NULL
+    AND "Permissions" ~ '^\s*\{.*\}\s*$'  -- Check if Permissions is a valid JSON object
+    AND jsonb_typeof("Permissions"::jsonb) = 'object'
+    AND (
+        ("Permissions"::jsonb)->>'editAssignedCollections' = 'true'
+        OR ("Permissions"::jsonb)->>'deleteAssignedCollections' = 'true'
+    )
+    AND ("Permissions"::jsonb)->>'accessEventLogs' = 'false'
+    AND ("Permissions"::jsonb)->>'accessImportExport' = 'false'
+    AND ("Permissions"::jsonb)->>'accessReports' = 'false'
+    AND ("Permissions"::jsonb)->>'createNewCollections' = 'false'
+    AND ("Permissions"::jsonb)->>'editAnyCollection' = 'false'
+    AND ("Permissions"::jsonb)->>'deleteAnyCollection' = 'false'
+    AND ("Permissions"::jsonb)->>'manageGroups' = 'false'
+    AND ("Permissions"::jsonb)->>'managePolicies' = 'false'
+    AND ("Permissions"::jsonb)->>'manageSso' = 'false'
+    AND ("Permissions"::jsonb)->>'manageUsers' = 'false'
+    AND ("Permissions"::jsonb)->>'manageResetPassword' = 'false'
+    AND ("Permissions"::jsonb)->>'manageScim' = 'false';
+
+-- Remove 'editAssignedCollections' and 'deleteAssignedCollections' properties from Permissions
+UPDATE "OrganizationUser"
+SET
+    "Permissions" = "Permissions"::jsonb - 'editAssignedCollections' - 'deleteAssignedCollections'
+WHERE
+    "Permissions" IS NOT NULL
+    AND "Permissions" ~ '^\s*\{.*\}\s*$'  -- Check if Permissions is a valid JSON object
+    AND jsonb_typeof("Permissions"::jsonb) = 'object';

util/PostgresMigrations/Migrations/20240828101424_FinalFlexibleCollectionsDataMigrations.cs

@@ -0,0 +1,21 @@
+using Bit.Core.Utilities;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Bit.PostgresMigrations.Migrations;
+
+public partial class FinalFlexibleCollectionsDataMigrations : Migration
+{
+    private const string _finalFlexibleCollectionsDataMigrationsScript = "PostgresMigrations.HelperScripts.2024-08-26_00_FinalFlexibleCollectionsDataMigrations.psql";
+
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql(CoreHelpers.GetEmbeddedResourceContentsAsync(_finalFlexibleCollectionsDataMigrationsScript));
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        throw new Exception("Irreversible migration");
+    }
+}

util/PostgresMigrations/PostgresMigrations.csproj

@@ -25,5 +25,6 @@
     <EmbeddedResource Include="HelperScripts\2022-03-01_00_Up_MigrateOrganizationApiKeys.psql" />
     <EmbeddedResource Include="HelperScripts\2022-03-01_00_Down_MigrateOrganizationApiKeys.psql" />
     <EmbeddedResource Include="HelperScripts\2024-04-25_00_EnableOrgsCollectionEnhancements.psql" />
+    <EmbeddedResource Include="HelperScripts\2024-08-26_00_FinalFlexibleCollectionsDataMigrations.psql" />
   </ItemGroup>
 </Project>

util/SqliteMigrations/HelperScripts/2024-08-26_00_FinalFlexibleCollectionsDataMigrations.sql

@@ -0,0 +1,38 @@
+-- Migrate Custom users who only have 'editAssignedCollections' and/or 'deleteAssignedCollections' custom permissions to the User type.
+UPDATE [OrganizationUser]
+SET
+    [Type] = 2,
+    [Permissions] = NULL
+WHERE
+    [Type] = 4
+    AND json_valid([Permissions]) = 1
+    AND (
+        json_extract([Permissions], '$.editAssignedCollections') = 1
+        OR json_extract([Permissions], '$.deleteAssignedCollections') = 1
+    )
+    AND json_extract([Permissions], '$.accessEventLogs') = 0
+    AND json_extract([Permissions], '$.accessImportExport') = 0
+    AND json_extract([Permissions], '$.accessReports') = 0
+    AND json_extract([Permissions], '$.createNewCollections') = 0
+    AND json_extract([Permissions], '$.editAnyCollection') = 0
+    AND json_extract([Permissions], '$.deleteAnyCollection') = 0
+    AND json_extract([Permissions], '$.manageGroups') = 0
+    AND json_extract([Permissions], '$.managePolicies') = 0
+    AND json_extract([Permissions], '$.manageSso') = 0
+    AND json_extract([Permissions], '$.manageUsers') = 0
+    AND json_extract([Permissions], '$.manageResetPassword') = 0
+    AND json_extract([Permissions], '$.manageScim') = 0;
+
+-- Remove 'editAssignedCollections' and 'deleteAssignedCollections' properties from Permissions
+UPDATE [OrganizationUser]
+SET
+    [Permissions] = json_remove(
+        json_remove([Permissions], '$.editAssignedCollections'),
+        '$.deleteAssignedCollections'
+    )
+WHERE
+    json_valid([Permissions]) = 1
+    AND (
+        json_extract([Permissions], '$.editAssignedCollections') IS NOT NULL
+        OR json_extract([Permissions], '$.deleteAssignedCollections') IS NOT NULL
+    );

util/SqliteMigrations/Migrations/20240828101418_FinalFlexibleCollectionsDataMigrations.cs

@@ -0,0 +1,21 @@
+using Bit.Core.Utilities;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Bit.SqliteMigrations.Migrations;
+
+public partial class FinalFlexibleCollectionsDataMigrations : Migration
+{
+    private const string _finalFlexibleCollectionsDataMigrationsScript = "SqliteMigrations.HelperScripts.2024-08-26_00_FinalFlexibleCollectionsDataMigrations.sql";
+
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql(CoreHelpers.GetEmbeddedResourceContentsAsync(_finalFlexibleCollectionsDataMigrationsScript));
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        throw new Exception("Irreversible migration");
+    }
+}

util/SqliteMigrations/SqliteMigrations.csproj

@@ -25,6 +25,7 @@
     <EmbeddedResource Include="HelperScripts\2023-12-04_00_Up_GrantIndexes.sql" />
     <EmbeddedResource Include="HelperScripts\2023-12-04_00_Down_GrantIndexes.sql" />
     <EmbeddedResource Include="HelperScripts\2024-04-25_00_EnableOrgsCollectionEnhancements.sql" />
+    <EmbeddedResource Include="HelperScripts\2024-08-26_00_FinalFlexibleCollectionsDataMigrations.sql" />
   </ItemGroup>
 
 </Project>