From 743465273ce18e8b6610eae25db35e0437e105b4 Mon Sep 17 00:00:00 2001 From: Matt Bishop Date: Fri, 22 Mar 2024 10:54:13 -0400 Subject: [PATCH] [PM-6909] Centralize database migration logic (#3910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Centralize database migration logic * Clean up unused usings * Prizatize * Remove verbose flag from Docker invocation * Allow argument passthrough still Co-authored-by: Michał Chęciński * Allow DI logger --------- Co-authored-by: Michał Chęciński --- util/Migrator/DbMigrator.cs | 76 ++++++++++++-------- util/Migrator/SqlServerDbMigrator.cs | 101 ++------------------------- util/MsSqlMigratorUtility/Dockerfile | 2 +- util/MsSqlMigratorUtility/Program.cs | 53 +++----------- util/Setup/Program.cs | 15 ++-- 5 files changed, 71 insertions(+), 176 deletions(-) diff --git a/util/Migrator/DbMigrator.cs b/util/Migrator/DbMigrator.cs index 24e78aaee..49aec7f40 100644 --- a/util/Migrator/DbMigrator.cs +++ b/util/Migrator/DbMigrator.cs @@ -12,39 +12,34 @@ public class DbMigrator { private readonly string _connectionString; private readonly ILogger _logger; - private readonly string _masterConnectionString; - public DbMigrator(string connectionString, ILogger logger) + public DbMigrator(string connectionString, ILogger logger = null) { _connectionString = connectionString; - _logger = logger; - _masterConnectionString = new SqlConnectionStringBuilder(connectionString) - { - InitialCatalog = "master" - }.ConnectionString; + _logger = logger ?? CreateLogger(); } public bool MigrateMsSqlDatabaseWithRetries(bool enableLogging = true, bool repeatable = false, string folderName = MigratorConstants.DefaultMigrationsFolderName, - CancellationToken cancellationToken = default(CancellationToken)) + CancellationToken cancellationToken = default) { var attempt = 1; - while (attempt < 10) { try { + PrepareDatabase(cancellationToken); + var success = MigrateDatabase(enableLogging, repeatable, folderName, cancellationToken); return success; } catch (SqlException ex) { - if (ex.Message.Contains("Server is in script upgrade mode")) + if (ex.Message.Contains("Server is in script upgrade mode.")) { attempt++; - _logger.LogInformation("Database is in script upgrade mode. " + - $"Trying again (attempt #{attempt})..."); + _logger.LogInformation($"Database is in script upgrade mode, trying again (attempt #{attempt})."); Thread.Sleep(20000); } else @@ -56,17 +51,14 @@ public class DbMigrator return false; } - public bool MigrateDatabase(bool enableLogging = true, - bool repeatable = false, - string folderName = MigratorConstants.DefaultMigrationsFolderName, - CancellationToken cancellationToken = default(CancellationToken)) + private void PrepareDatabase(CancellationToken cancellationToken = default) { - if (_logger != null) + var masterConnectionString = new SqlConnectionStringBuilder(_connectionString) { - _logger.LogInformation(Constants.BypassFiltersEventId, "Migrating database."); - } + InitialCatalog = "master" + }.ConnectionString; - using (var connection = new SqlConnection(_masterConnectionString)) + using (var connection = new SqlConnection(masterConnectionString)) { var databaseName = new SqlConnectionStringBuilder(_connectionString).InitialCatalog; if (string.IsNullOrWhiteSpace(databaseName)) @@ -89,9 +81,10 @@ public class DbMigrator } cancellationToken.ThrowIfCancellationRequested(); + using (var connection = new SqlConnection(_connectionString)) { - // Rename old migration scripts to new namespace. + // rename old migration scripts to new namespace var command = new SqlCommand( "IF OBJECT_ID('Migration','U') IS NOT NULL " + "UPDATE [dbo].[Migration] SET " + @@ -101,6 +94,20 @@ public class DbMigrator } cancellationToken.ThrowIfCancellationRequested(); + } + + private bool MigrateDatabase(bool enableLogging = true, + bool repeatable = false, + string folderName = MigratorConstants.DefaultMigrationsFolderName, + CancellationToken cancellationToken = default) + { + if (enableLogging) + { + _logger.LogInformation(Constants.BypassFiltersEventId, "Migrating database."); + } + + cancellationToken.ThrowIfCancellationRequested(); + var builder = DeployChanges.To .SqlDatabase(_connectionString) .WithScriptsAndCodeEmbeddedInAssembly(Assembly.GetExecutingAssembly(), @@ -119,20 +126,13 @@ public class DbMigrator if (enableLogging) { - if (_logger != null) - { - builder.LogTo(new DbUpLogger(_logger)); - } - else - { - builder.LogToConsole(); - } + builder.LogTo(new DbUpLogger(_logger)); } var upgrader = builder.Build(); var result = upgrader.PerformUpgrade(); - if (_logger != null) + if (enableLogging) { if (result.Successful) { @@ -145,6 +145,22 @@ public class DbMigrator } cancellationToken.ThrowIfCancellationRequested(); + return result.Successful; } + + private ILogger CreateLogger() + { + var loggerFactory = LoggerFactory.Create(builder => + { + builder + .AddFilter("Microsoft", LogLevel.Warning) + .AddFilter("System", LogLevel.Warning) + .AddConsole(); + + builder.AddFilter("DbMigrator.DbMigrator", LogLevel.Information); + }); + + return loggerFactory.CreateLogger(); + } } diff --git a/util/Migrator/SqlServerDbMigrator.cs b/util/Migrator/SqlServerDbMigrator.cs index 3885a6f6c..b44326082 100644 --- a/util/Migrator/SqlServerDbMigrator.cs +++ b/util/Migrator/SqlServerDbMigrator.cs @@ -1,109 +1,22 @@ -using System.Data; -using System.Reflection; -using Bit.Core; -using Bit.Core.Settings; +using Bit.Core.Settings; using Bit.Core.Utilities; -using DbUp; -using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; namespace Bit.Migrator; public class SqlServerDbMigrator : IDbMigrator { - private readonly string _connectionString; - private readonly ILogger _logger; - private readonly string _masterConnectionString; + private readonly DbMigrator _migrator; - public SqlServerDbMigrator(GlobalSettings globalSettings, ILogger logger) + public SqlServerDbMigrator(GlobalSettings globalSettings, ILogger logger) { - _connectionString = globalSettings.SqlServer.ConnectionString; - _logger = logger; - _masterConnectionString = new SqlConnectionStringBuilder(_connectionString) - { - InitialCatalog = "master" - }.ConnectionString; + _migrator = new DbMigrator(globalSettings.SqlServer.ConnectionString, logger); } public bool MigrateDatabase(bool enableLogging = true, - CancellationToken cancellationToken = default(CancellationToken)) + CancellationToken cancellationToken = default) { - if (enableLogging && _logger != null) - { - _logger.LogInformation(Constants.BypassFiltersEventId, "Migrating database."); - } - - using (var connection = new SqlConnection(_masterConnectionString)) - { - var databaseName = new SqlConnectionStringBuilder(_connectionString).InitialCatalog; - if (string.IsNullOrWhiteSpace(databaseName)) - { - databaseName = "vault"; - } - - var databaseNameQuoted = new SqlCommandBuilder().QuoteIdentifier(databaseName); - var command = new SqlCommand( - "IF ((SELECT COUNT(1) FROM sys.databases WHERE [name] = @DatabaseName) = 0) " + - "CREATE DATABASE " + databaseNameQuoted + ";", connection); - command.Parameters.Add("@DatabaseName", SqlDbType.VarChar).Value = databaseName; - command.Connection.Open(); - command.ExecuteNonQuery(); - - command.CommandText = "IF ((SELECT DATABASEPROPERTYEX([name], 'IsAutoClose') " + - "FROM sys.databases WHERE [name] = @DatabaseName) = 1) " + - "ALTER DATABASE " + databaseNameQuoted + " SET AUTO_CLOSE OFF;"; - command.ExecuteNonQuery(); - } - - cancellationToken.ThrowIfCancellationRequested(); - using (var connection = new SqlConnection(_connectionString)) - { - // Rename old migration scripts to new namespace. - var command = new SqlCommand( - "IF OBJECT_ID('Migration','U') IS NOT NULL " + - "UPDATE [dbo].[Migration] SET " + - "[ScriptName] = REPLACE([ScriptName], 'Bit.Setup.', 'Bit.Migrator.');", connection); - command.Connection.Open(); - command.ExecuteNonQuery(); - } - - cancellationToken.ThrowIfCancellationRequested(); - var builder = DeployChanges.To - .SqlDatabase(_connectionString) - .JournalToSqlTable("dbo", MigratorConstants.SqlTableJournalName) - .WithScriptsAndCodeEmbeddedInAssembly(Assembly.GetExecutingAssembly(), - s => s.Contains($".DbScripts.") && !s.Contains(".Archive.")) - .WithTransaction() - .WithExecutionTimeout(TimeSpan.FromMinutes(5)); - - if (enableLogging) - { - if (_logger != null) - { - builder.LogTo(new DbUpLogger(_logger)); - } - else - { - builder.LogToConsole(); - } - } - - var upgrader = builder.Build(); - var result = upgrader.PerformUpgrade(); - - if (enableLogging && _logger != null) - { - if (result.Successful) - { - _logger.LogInformation(Constants.BypassFiltersEventId, "Migration successful."); - } - else - { - _logger.LogError(Constants.BypassFiltersEventId, result.Error, "Migration failed."); - } - } - - cancellationToken.ThrowIfCancellationRequested(); - return result.Successful; + return _migrator.MigrateMsSqlDatabaseWithRetries(enableLogging, + cancellationToken: cancellationToken); } } diff --git a/util/MsSqlMigratorUtility/Dockerfile b/util/MsSqlMigratorUtility/Dockerfile index 7b53905f4..b3da6a53f 100644 --- a/util/MsSqlMigratorUtility/Dockerfile +++ b/util/MsSqlMigratorUtility/Dockerfile @@ -5,4 +5,4 @@ LABEL com.bitwarden.product="bitwarden" WORKDIR /app COPY obj/build-output/publish . -ENTRYPOINT ["sh", "-c", "dotnet /app/MsSqlMigratorUtility.dll \"${MSSQL_CONN_STRING}\" -v ${@}", "--" ] +ENTRYPOINT ["sh", "-c", "dotnet /app/MsSqlMigratorUtility.dll \"${MSSQL_CONN_STRING}\" ${@}", "--" ] diff --git a/util/MsSqlMigratorUtility/Program.cs b/util/MsSqlMigratorUtility/Program.cs index 681225ca3..5f215d155 100644 --- a/util/MsSqlMigratorUtility/Program.cs +++ b/util/MsSqlMigratorUtility/Program.cs @@ -1,11 +1,8 @@ using Bit.Migrator; using CommandDotNet; -using Microsoft.Extensions.Logging; internal class Program { - private static IDictionary Parameters { get; set; } - private static int Main(string[] args) { return new AppRunner().Run(args); @@ -15,60 +12,26 @@ internal class Program public void Execute( [Operand(Description = "Database connection string")] string databaseConnectionString, - [Option('v', "verbose", Description = "Enable verbose output of migrator logs")] - bool verbose = false, [Option('r', "repeatable", Description = "Mark scripts as repeatable")] bool repeatable = false, [Option('f', "folder", Description = "Folder name of database scripts")] - string folderName = MigratorConstants.DefaultMigrationsFolderName) => MigrateDatabase(databaseConnectionString, verbose, repeatable, folderName); + string folderName = MigratorConstants.DefaultMigrationsFolderName) + => MigrateDatabase(databaseConnectionString, repeatable, folderName); - private static void WriteUsageToConsole() + private static bool MigrateDatabase(string databaseConnectionString, + bool repeatable = false, string folderName = "") { - Console.WriteLine("Usage: MsSqlMigratorUtility "); - Console.WriteLine("Usage: MsSqlMigratorUtility -v|--verbose (for verbose output of migrator logs)"); - Console.WriteLine("Usage: MsSqlMigratorUtility -r|--repeatable (for marking scripts as repeatable) -f|--folder (for specifying folder name of scripts)"); - Console.WriteLine("Usage: MsSqlMigratorUtility -v|--verbose (for verbose output of migrator logs) -r|--repeatable (for marking scripts as repeatable) -f|--folder (for specifying folder name of scripts)"); - } - - private static bool MigrateDatabase(string databaseConnectionString, bool verbose = false, bool repeatable = false, string folderName = "") - { - var logger = CreateLogger(verbose); - - logger.LogInformation($"Migrating database with repeatable: {repeatable} and folderName: {folderName}."); - - var migrator = new DbMigrator(databaseConnectionString, logger); - bool success = false; + var migrator = new DbMigrator(databaseConnectionString); + bool success; if (!string.IsNullOrWhiteSpace(folderName)) { - success = migrator.MigrateMsSqlDatabaseWithRetries(verbose, repeatable, folderName); + success = migrator.MigrateMsSqlDatabaseWithRetries(true, repeatable, folderName); } else { - success = migrator.MigrateMsSqlDatabaseWithRetries(verbose, repeatable); + success = migrator.MigrateMsSqlDatabaseWithRetries(true, repeatable); } return success; } - - private static ILogger CreateLogger(bool verbose) - { - var loggerFactory = LoggerFactory.Create(builder => - { - builder - .AddFilter("Microsoft", LogLevel.Warning) - .AddFilter("System", LogLevel.Warning) - .AddConsole(); - - if (verbose) - { - builder.AddFilter("DbMigrator.DbMigrator", LogLevel.Debug); - } - else - { - builder.AddFilter("DbMigrator.DbMigrator", LogLevel.Information); - } - }); - var logger = loggerFactory.CreateLogger(); - return logger; - } } diff --git a/util/Setup/Program.cs b/util/Setup/Program.cs index 93304c6bb..5768db7ab 100644 --- a/util/Setup/Program.cs +++ b/util/Setup/Program.cs @@ -17,6 +17,7 @@ public class Program { Args = args }; + ParseParameters(); if (_context.Parameters.ContainsKey("q")) @@ -155,7 +156,7 @@ public class Program if (_context.Parameters.ContainsKey("db")) { - MigrateDatabase(); + PrepareAndMigrateDatabase(); } else { @@ -185,17 +186,19 @@ public class Program Console.WriteLine("\n"); } - private static void MigrateDatabase(int attempt = 1) + private static void PrepareAndMigrateDatabase() { var vaultConnectionString = Helpers.GetValueFromEnvFile("global", "globalSettings__sqlServer__connectionString"); - var migrator = new DbMigrator(vaultConnectionString, null); + var migrator = new DbMigrator(vaultConnectionString); - var log = false; + var enableLogging = false; - migrator.MigrateMsSqlDatabaseWithRetries(log); + // execute all general migration scripts (will detect those not yet applied) + migrator.MigrateMsSqlDatabaseWithRetries(enableLogging); - migrator.MigrateMsSqlDatabaseWithRetries(log, true, MigratorConstants.TransitionMigrationsFolderName); + // execute explicit transition migration scripts, per EDD + migrator.MigrateMsSqlDatabaseWithRetries(enableLogging, true, MigratorConstants.TransitionMigrationsFolderName); } private static bool ValidateInstallation()