[PM-6909] Centralize database migration logic (#3910)

* Centralize database migration logic

* Clean up unused usings

* Prizatize

* Remove verbose flag from Docker invocation

* Allow argument passthrough still

Co-authored-by: Michał Chęciński <mchecinski@bitwarden.com>

* Allow DI logger

---------

Co-authored-by: Michał Chęciński <mchecinski@bitwarden.com>

util/Migrator/DbMigrator.cs

@@ -12,39 +12,34 @@ public class DbMigrator
 {
     private readonly string _connectionString;
     private readonly ILogger<DbMigrator> _logger;
-    private readonly string _masterConnectionString;
 
-    public DbMigrator(string connectionString, ILogger<DbMigrator> logger)
+    public DbMigrator(string connectionString, ILogger<DbMigrator> logger = null)
     {
         _connectionString = connectionString;
-        _logger = logger;
+        _logger = logger ?? CreateLogger();
-        _masterConnectionString = new SqlConnectionStringBuilder(connectionString)
-        {
-            InitialCatalog = "master"
-        }.ConnectionString;
     }
 
     public bool MigrateMsSqlDatabaseWithRetries(bool enableLogging = true,
         bool repeatable = false,
         string folderName = MigratorConstants.DefaultMigrationsFolderName,
-        CancellationToken cancellationToken = default(CancellationToken))
+        CancellationToken cancellationToken = default)
     {
         var attempt = 1;
-
         while (attempt < 10)
         {
             try
             {
+                PrepareDatabase(cancellationToken);
+
                 var success = MigrateDatabase(enableLogging, repeatable, folderName, cancellationToken);
                 return success;
             }
             catch (SqlException ex)
             {
-                if (ex.Message.Contains("Server is in script upgrade mode"))
+                if (ex.Message.Contains("Server is in script upgrade mode."))
                 {
                     attempt++;
-                    _logger.LogInformation("Database is in script upgrade mode. " +
+                    _logger.LogInformation($"Database is in script upgrade mode, trying again (attempt #{attempt}).");
-                        $"Trying again (attempt #{attempt})...");
                     Thread.Sleep(20000);
                 }
                 else
@@ -56,17 +51,14 @@ public class DbMigrator
         return false;
     }
 
-    public bool MigrateDatabase(bool enableLogging = true,
+    private void PrepareDatabase(CancellationToken cancellationToken = default)
-        bool repeatable = false,
-        string folderName = MigratorConstants.DefaultMigrationsFolderName,
-        CancellationToken cancellationToken = default(CancellationToken))
     {
-        if (_logger != null)
+        var masterConnectionString = new SqlConnectionStringBuilder(_connectionString)
         {
-            _logger.LogInformation(Constants.BypassFiltersEventId, "Migrating database.");
+            InitialCatalog = "master"
-        }
+        }.ConnectionString;
 
-        using (var connection = new SqlConnection(_masterConnectionString))
+        using (var connection = new SqlConnection(masterConnectionString))
         {
             var databaseName = new SqlConnectionStringBuilder(_connectionString).InitialCatalog;
             if (string.IsNullOrWhiteSpace(databaseName))
@@ -89,9 +81,10 @@ public class DbMigrator
         }
 
         cancellationToken.ThrowIfCancellationRequested();
+
         using (var connection = new SqlConnection(_connectionString))
         {
-            // Rename old migration scripts to new namespace.
+            // rename old migration scripts to new namespace
             var command = new SqlCommand(
                 "IF OBJECT_ID('Migration','U') IS NOT NULL " +
                 "UPDATE [dbo].[Migration] SET " +
@@ -101,6 +94,20 @@ public class DbMigrator
         }
 
         cancellationToken.ThrowIfCancellationRequested();
+    }
+
+    private bool MigrateDatabase(bool enableLogging = true,
+        bool repeatable = false,
+        string folderName = MigratorConstants.DefaultMigrationsFolderName,
+        CancellationToken cancellationToken = default)
+    {
+        if (enableLogging)
+        {
+            _logger.LogInformation(Constants.BypassFiltersEventId, "Migrating database.");
+        }
+
+        cancellationToken.ThrowIfCancellationRequested();
+
         var builder = DeployChanges.To
             .SqlDatabase(_connectionString)
             .WithScriptsAndCodeEmbeddedInAssembly(Assembly.GetExecutingAssembly(),
@@ -119,20 +126,13 @@ public class DbMigrator
 
         if (enableLogging)
         {
-            if (_logger != null)
+            builder.LogTo(new DbUpLogger(_logger));
-            {
-                builder.LogTo(new DbUpLogger(_logger));
-            }
-            else
-            {
-                builder.LogToConsole();
-            }
         }
 
         var upgrader = builder.Build();
         var result = upgrader.PerformUpgrade();
 
-        if (_logger != null)
+        if (enableLogging)
         {
             if (result.Successful)
             {
@@ -145,6 +145,22 @@ public class DbMigrator
         }
 
         cancellationToken.ThrowIfCancellationRequested();
+
         return result.Successful;
     }
+
+    private ILogger<DbMigrator> CreateLogger()
+    {
+        var loggerFactory = LoggerFactory.Create(builder =>
+        {
+            builder
+                .AddFilter("Microsoft", LogLevel.Warning)
+                .AddFilter("System", LogLevel.Warning)
+                .AddConsole();
+
+            builder.AddFilter("DbMigrator.DbMigrator", LogLevel.Information);
+        });
+
+        return loggerFactory.CreateLogger<DbMigrator>();
+    }
 }

util/Migrator/SqlServerDbMigrator.cs

@@ -1,109 +1,22 @@
-using System.Data;
+using Bit.Core.Settings;
-using System.Reflection;
-using Bit.Core;
-using Bit.Core.Settings;
 using Bit.Core.Utilities;
-using DbUp;
-using Microsoft.Data.SqlClient;
 using Microsoft.Extensions.Logging;
 
 namespace Bit.Migrator;
 
 public class SqlServerDbMigrator : IDbMigrator
 {
-    private readonly string _connectionString;
+    private readonly DbMigrator _migrator;
-    private readonly ILogger<SqlServerDbMigrator> _logger;
-    private readonly string _masterConnectionString;
 
-    public SqlServerDbMigrator(GlobalSettings globalSettings, ILogger<SqlServerDbMigrator> logger)
+    public SqlServerDbMigrator(GlobalSettings globalSettings, ILogger<DbMigrator> logger)
     {
-        _connectionString = globalSettings.SqlServer.ConnectionString;
+        _migrator = new DbMigrator(globalSettings.SqlServer.ConnectionString, logger);
-        _logger = logger;
-        _masterConnectionString = new SqlConnectionStringBuilder(_connectionString)
-        {
-            InitialCatalog = "master"
-        }.ConnectionString;
     }
 
     public bool MigrateDatabase(bool enableLogging = true,
-        CancellationToken cancellationToken = default(CancellationToken))
+        CancellationToken cancellationToken = default)
     {
-        if (enableLogging && _logger != null)
+        return _migrator.MigrateMsSqlDatabaseWithRetries(enableLogging,
-        {
+            cancellationToken: cancellationToken);
-            _logger.LogInformation(Constants.BypassFiltersEventId, "Migrating database.");
-        }
-
-        using (var connection = new SqlConnection(_masterConnectionString))
-        {
-            var databaseName = new SqlConnectionStringBuilder(_connectionString).InitialCatalog;
-            if (string.IsNullOrWhiteSpace(databaseName))
-            {
-                databaseName = "vault";
-            }
-
-            var databaseNameQuoted = new SqlCommandBuilder().QuoteIdentifier(databaseName);
-            var command = new SqlCommand(
-                "IF ((SELECT COUNT(1) FROM sys.databases WHERE [name] = @DatabaseName) = 0) " +
-                "CREATE DATABASE " + databaseNameQuoted + ";", connection);
-            command.Parameters.Add("@DatabaseName", SqlDbType.VarChar).Value = databaseName;
-            command.Connection.Open();
-            command.ExecuteNonQuery();
-
-            command.CommandText = "IF ((SELECT DATABASEPROPERTYEX([name], 'IsAutoClose') " +
-                "FROM sys.databases WHERE [name] = @DatabaseName) = 1) " +
-                "ALTER DATABASE " + databaseNameQuoted + " SET AUTO_CLOSE OFF;";
-            command.ExecuteNonQuery();
-        }
-
-        cancellationToken.ThrowIfCancellationRequested();
-        using (var connection = new SqlConnection(_connectionString))
-        {
-            // Rename old migration scripts to new namespace.
-            var command = new SqlCommand(
-                "IF OBJECT_ID('Migration','U') IS NOT NULL " +
-                "UPDATE [dbo].[Migration] SET " +
-                "[ScriptName] = REPLACE([ScriptName], 'Bit.Setup.', 'Bit.Migrator.');", connection);
-            command.Connection.Open();
-            command.ExecuteNonQuery();
-        }
-
-        cancellationToken.ThrowIfCancellationRequested();
-        var builder = DeployChanges.To
-            .SqlDatabase(_connectionString)
-            .JournalToSqlTable("dbo", MigratorConstants.SqlTableJournalName)
-            .WithScriptsAndCodeEmbeddedInAssembly(Assembly.GetExecutingAssembly(),
-                s => s.Contains($".DbScripts.") && !s.Contains(".Archive."))
-            .WithTransaction()
-            .WithExecutionTimeout(TimeSpan.FromMinutes(5));
-
-        if (enableLogging)
-        {
-            if (_logger != null)
-            {
-                builder.LogTo(new DbUpLogger(_logger));
-            }
-            else
-            {
-                builder.LogToConsole();
-            }
-        }
-
-        var upgrader = builder.Build();
-        var result = upgrader.PerformUpgrade();
-
-        if (enableLogging && _logger != null)
-        {
-            if (result.Successful)
-            {
-                _logger.LogInformation(Constants.BypassFiltersEventId, "Migration successful.");
-            }
-            else
-            {
-                _logger.LogError(Constants.BypassFiltersEventId, result.Error, "Migration failed.");
-            }
-        }
-
-        cancellationToken.ThrowIfCancellationRequested();
-        return result.Successful;
     }
 }

util/MsSqlMigratorUtility/Dockerfile

@@ -5,4 +5,4 @@ LABEL com.bitwarden.product="bitwarden"
 WORKDIR /app
 COPY obj/build-output/publish .
 
-ENTRYPOINT ["sh", "-c", "dotnet /app/MsSqlMigratorUtility.dll \"${MSSQL_CONN_STRING}\" -v ${@}", "--" ]
+ENTRYPOINT ["sh", "-c", "dotnet /app/MsSqlMigratorUtility.dll \"${MSSQL_CONN_STRING}\" ${@}", "--" ]

util/MsSqlMigratorUtility/Program.cs

@@ -1,11 +1,8 @@
 using Bit.Migrator;
 using CommandDotNet;
-using Microsoft.Extensions.Logging;
 
 internal class Program
 {
-    private static IDictionary<string, string> Parameters { get; set; }
-
     private static int Main(string[] args)
     {
         return new AppRunner<Program>().Run(args);
@@ -15,60 +12,26 @@ internal class Program
     public void Execute(
         [Operand(Description = "Database connection string")]
         string databaseConnectionString,
-        [Option('v', "verbose", Description = "Enable verbose output of migrator logs")]
-        bool verbose = false,
         [Option('r', "repeatable", Description = "Mark scripts as repeatable")]
         bool repeatable = false,
         [Option('f', "folder", Description = "Folder name of database scripts")]
-        string folderName = MigratorConstants.DefaultMigrationsFolderName) => MigrateDatabase(databaseConnectionString, verbose, repeatable, folderName);
+        string folderName = MigratorConstants.DefaultMigrationsFolderName)
+        => MigrateDatabase(databaseConnectionString, repeatable, folderName);
 
-    private static void WriteUsageToConsole()
+    private static bool MigrateDatabase(string databaseConnectionString,
+        bool repeatable = false, string folderName = "")
     {
-        Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string>");
+        var migrator = new DbMigrator(databaseConnectionString);
-        Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string> -v|--verbose (for verbose output of migrator logs)");
+        bool success;
-        Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string> -r|--repeatable (for marking scripts as repeatable) -f|--folder <folder-name-in-migrator-project> (for specifying folder name of scripts)");
-        Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string> -v|--verbose (for verbose output of migrator logs) -r|--repeatable (for marking scripts as repeatable) -f|--folder <folder-name-in-migrator-project> (for specifying folder name of scripts)");
-    }
-
-    private static bool MigrateDatabase(string databaseConnectionString, bool verbose = false, bool repeatable = false, string folderName = "")
-    {
-        var logger = CreateLogger(verbose);
-
-        logger.LogInformation($"Migrating database with repeatable: {repeatable} and folderName: {folderName}.");
-
-        var migrator = new DbMigrator(databaseConnectionString, logger);
-        bool success = false;
         if (!string.IsNullOrWhiteSpace(folderName))
         {
-            success = migrator.MigrateMsSqlDatabaseWithRetries(verbose, repeatable, folderName);
+            success = migrator.MigrateMsSqlDatabaseWithRetries(true, repeatable, folderName);
         }
         else
         {
-            success = migrator.MigrateMsSqlDatabaseWithRetries(verbose, repeatable);
+            success = migrator.MigrateMsSqlDatabaseWithRetries(true, repeatable);
         }
 
         return success;
     }
-
-    private static ILogger<DbMigrator> CreateLogger(bool verbose)
-    {
-        var loggerFactory = LoggerFactory.Create(builder =>
-        {
-            builder
-                .AddFilter("Microsoft", LogLevel.Warning)
-                .AddFilter("System", LogLevel.Warning)
-                .AddConsole();
-
-            if (verbose)
-            {
-                builder.AddFilter("DbMigrator.DbMigrator", LogLevel.Debug);
-            }
-            else
-            {
-                builder.AddFilter("DbMigrator.DbMigrator", LogLevel.Information);
-            }
-        });
-        var logger = loggerFactory.CreateLogger<DbMigrator>();
-        return logger;
-    }
 }

util/Setup/Program.cs

@@ -17,6 +17,7 @@ public class Program
         {
             Args = args
         };
+
         ParseParameters();
 
         if (_context.Parameters.ContainsKey("q"))
@@ -155,7 +156,7 @@ public class Program
 
         if (_context.Parameters.ContainsKey("db"))
         {
-            MigrateDatabase();
+            PrepareAndMigrateDatabase();
         }
         else
         {
@@ -185,17 +186,19 @@ public class Program
         Console.WriteLine("\n");
     }
 
-    private static void MigrateDatabase(int attempt = 1)
+    private static void PrepareAndMigrateDatabase()
     {
         var vaultConnectionString = Helpers.GetValueFromEnvFile("global",
             "globalSettings__sqlServer__connectionString");
-        var migrator = new DbMigrator(vaultConnectionString, null);
+        var migrator = new DbMigrator(vaultConnectionString);
 
-        var log = false;
+        var enableLogging = false;
 
-        migrator.MigrateMsSqlDatabaseWithRetries(log);
+        // execute all general migration scripts (will detect those not yet applied)
+        migrator.MigrateMsSqlDatabaseWithRetries(enableLogging);
 
-        migrator.MigrateMsSqlDatabaseWithRetries(log, true, MigratorConstants.TransitionMigrationsFolderName);
+        // execute explicit transition migration scripts, per EDD
+        migrator.MigrateMsSqlDatabaseWithRetries(enableLogging, true, MigratorConstants.TransitionMigrationsFolderName);
     }
 
     private static bool ValidateInstallation()