From 3e90f57797ba157a056a27051e4e1ab6c3ae1ff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ch=C4=99ci=C5=84ski?= Date: Thu, 28 Sep 2023 16:29:52 +0200 Subject: [PATCH] [DEVOPS-1519] Add transition mode to mssql migrator utility (#3259) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add RerunableSqlTableJournal * Add extension to use rerunable sql table journal * Use rerunable sql journal * format * Enable logging * FIx * Disable logging * Rename to SqlTableJournalExtensions * Move RerunableSqlTableJournal to Extension class * Fix usings * Add rerunable schema * Format * Fix typo * Enable logging in db migrator * add rerunable column in dbo migrations table migration * Trying * Fix journal table name * Trying to migrate first * After migration * Testing * Add update from rerunable to not rerunable script * Change name * Add rerunable option and script folder name * Add rerunable options and folder * Fix * Add transition (aka rerunable) migrations to Setup * Parse parameters on migrator utility * Fix sql scripts * Remove CreateSchemaTableSql as it'll be migrated using migration * Embed dbScripts_data_migration folder * Remove testing sql script * Add optins parsing nuget for msSqlMigratorUtility * Fix sql journal * Ran dotnet format * Comment out index * ▫️Revert "Comment out index" This reverts commit df15fa91e05d5b195e130c36e5ae51fc508b2506. * Disable logging * Add newline * Rename rerunable to repeatable * remove repeatable journal * Remove migration adding the repeatable column in dbo.Migrations table * Add using * Enable log for testing * Disable logging in the setup * Remove unused method * Add migrator constants * Use constants in yet another place * Fix * Add constant * Fix * Fix --- util/Migrator/DbMigrator.cs | 19 +++++- util/Migrator/Migrator.csproj | 1 + util/Migrator/MigratorConstants.cs | 8 +++ util/Migrator/SqlServerDbMigrator.cs | 2 +- .../MsSqlMigratorUtility.csproj | 1 + util/MsSqlMigratorUtility/Program.cs | 62 +++++++++---------- util/MsSqlMigratorUtility/packages.lock.json | 9 +++ util/Setup/Program.cs | 7 ++- 8 files changed, 71 insertions(+), 38 deletions(-) create mode 100644 util/Migrator/MigratorConstants.cs diff --git a/util/Migrator/DbMigrator.cs b/util/Migrator/DbMigrator.cs index 2dca5006a..24e78aaee 100644 --- a/util/Migrator/DbMigrator.cs +++ b/util/Migrator/DbMigrator.cs @@ -2,6 +2,7 @@ using System.Reflection; using Bit.Core; using DbUp; +using DbUp.Helpers; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; @@ -24,6 +25,8 @@ public class DbMigrator } public bool MigrateMsSqlDatabaseWithRetries(bool enableLogging = true, + bool repeatable = false, + string folderName = MigratorConstants.DefaultMigrationsFolderName, CancellationToken cancellationToken = default(CancellationToken)) { var attempt = 1; @@ -32,7 +35,7 @@ public class DbMigrator { try { - var success = MigrateDatabase(enableLogging, cancellationToken); + var success = MigrateDatabase(enableLogging, repeatable, folderName, cancellationToken); return success; } catch (SqlException ex) @@ -54,6 +57,8 @@ public class DbMigrator } public bool MigrateDatabase(bool enableLogging = true, + bool repeatable = false, + string folderName = MigratorConstants.DefaultMigrationsFolderName, CancellationToken cancellationToken = default(CancellationToken)) { if (_logger != null) @@ -98,12 +103,20 @@ public class DbMigrator cancellationToken.ThrowIfCancellationRequested(); var builder = DeployChanges.To .SqlDatabase(_connectionString) - .JournalToSqlTable("dbo", "Migration") .WithScriptsAndCodeEmbeddedInAssembly(Assembly.GetExecutingAssembly(), - s => s.Contains($".DbScripts.") && !s.Contains(".Archive.")) + s => s.Contains($".{folderName}.") && !s.Contains(".Archive.")) .WithTransaction() .WithExecutionTimeout(new TimeSpan(0, 5, 0)); + if (repeatable) + { + builder.JournalTo(new NullJournal()); + } + else + { + builder.JournalToSqlTable("dbo", MigratorConstants.SqlTableJournalName); + } + if (enableLogging) { if (_logger != null) diff --git a/util/Migrator/Migrator.csproj b/util/Migrator/Migrator.csproj index c7f00465d..e18f5cc1f 100644 --- a/util/Migrator/Migrator.csproj +++ b/util/Migrator/Migrator.csproj @@ -2,6 +2,7 @@ + diff --git a/util/Migrator/MigratorConstants.cs b/util/Migrator/MigratorConstants.cs new file mode 100644 index 000000000..1ffcfcfe2 --- /dev/null +++ b/util/Migrator/MigratorConstants.cs @@ -0,0 +1,8 @@ +namespace Bit.Migrator; + +public static class MigratorConstants +{ + public const string SqlTableJournalName = "Migration"; + public const string DefaultMigrationsFolderName = "DbScripts"; + public const string TransitionMigrationsFolderName = "DbScripts_data_migration"; +} diff --git a/util/Migrator/SqlServerDbMigrator.cs b/util/Migrator/SqlServerDbMigrator.cs index 219a3ff39..3885a6f6c 100644 --- a/util/Migrator/SqlServerDbMigrator.cs +++ b/util/Migrator/SqlServerDbMigrator.cs @@ -70,7 +70,7 @@ public class SqlServerDbMigrator : IDbMigrator cancellationToken.ThrowIfCancellationRequested(); var builder = DeployChanges.To .SqlDatabase(_connectionString) - .JournalToSqlTable("dbo", "Migration") + .JournalToSqlTable("dbo", MigratorConstants.SqlTableJournalName) .WithScriptsAndCodeEmbeddedInAssembly(Assembly.GetExecutingAssembly(), s => s.Contains($".DbScripts.") && !s.Contains(".Archive.")) .WithTransaction() diff --git a/util/MsSqlMigratorUtility/MsSqlMigratorUtility.csproj b/util/MsSqlMigratorUtility/MsSqlMigratorUtility.csproj index 0e423b150..7f19d6745 100644 --- a/util/MsSqlMigratorUtility/MsSqlMigratorUtility.csproj +++ b/util/MsSqlMigratorUtility/MsSqlMigratorUtility.csproj @@ -10,6 +10,7 @@ + diff --git a/util/MsSqlMigratorUtility/Program.cs b/util/MsSqlMigratorUtility/Program.cs index 517b9ecd9..681225ca3 100644 --- a/util/MsSqlMigratorUtility/Program.cs +++ b/util/MsSqlMigratorUtility/Program.cs @@ -1,55 +1,51 @@ using Bit.Migrator; +using CommandDotNet; using Microsoft.Extensions.Logging; internal class Program { + private static IDictionary Parameters { get; set; } + private static int Main(string[] args) { - if (args.Length == 0) - { - Console.WriteLine("Please enter a database connection string argument."); - WriteUsageToConsole(); - return 1; - } - - if (args.Length == 1 && (args[0] == "--verbose" || args[0] == "-v")) - { - Console.WriteLine($"Please enter a database connection string argument before {args[0]} option."); - WriteUsageToConsole(); - return 1; - } - - var databaseConnectionString = args[0]; - - var verbose = false; - - if (args.Length == 2 && (args[1] == "--verbose" || args[1] == "-v")) - { - verbose = true; - } - - var success = MigrateDatabase(databaseConnectionString, verbose); - - if (!success) - { - return -1; - } - - return 0; + return new AppRunner().Run(args); } + [DefaultCommand] + public void Execute( + [Operand(Description = "Database connection string")] + string databaseConnectionString, + [Option('v', "verbose", Description = "Enable verbose output of migrator logs")] + bool verbose = false, + [Option('r', "repeatable", Description = "Mark scripts as repeatable")] + bool repeatable = false, + [Option('f', "folder", Description = "Folder name of database scripts")] + string folderName = MigratorConstants.DefaultMigrationsFolderName) => MigrateDatabase(databaseConnectionString, verbose, repeatable, folderName); + private static void WriteUsageToConsole() { Console.WriteLine("Usage: MsSqlMigratorUtility "); Console.WriteLine("Usage: MsSqlMigratorUtility -v|--verbose (for verbose output of migrator logs)"); + Console.WriteLine("Usage: MsSqlMigratorUtility -r|--repeatable (for marking scripts as repeatable) -f|--folder (for specifying folder name of scripts)"); + Console.WriteLine("Usage: MsSqlMigratorUtility -v|--verbose (for verbose output of migrator logs) -r|--repeatable (for marking scripts as repeatable) -f|--folder (for specifying folder name of scripts)"); } - private static bool MigrateDatabase(string databaseConnectionString, bool verbose = false, int attempt = 1) + private static bool MigrateDatabase(string databaseConnectionString, bool verbose = false, bool repeatable = false, string folderName = "") { var logger = CreateLogger(verbose); + logger.LogInformation($"Migrating database with repeatable: {repeatable} and folderName: {folderName}."); + var migrator = new DbMigrator(databaseConnectionString, logger); - var success = migrator.MigrateMsSqlDatabaseWithRetries(verbose); + bool success = false; + if (!string.IsNullOrWhiteSpace(folderName)) + { + success = migrator.MigrateMsSqlDatabaseWithRetries(verbose, repeatable, folderName); + } + else + { + success = migrator.MigrateMsSqlDatabaseWithRetries(verbose, repeatable); + } return success; } diff --git a/util/MsSqlMigratorUtility/packages.lock.json b/util/MsSqlMigratorUtility/packages.lock.json index e39912d7d..7a1316c0f 100644 --- a/util/MsSqlMigratorUtility/packages.lock.json +++ b/util/MsSqlMigratorUtility/packages.lock.json @@ -2,6 +2,15 @@ "version": 1, "dependencies": { "net6.0": { + "CommandDotNet": { + "type": "Direct", + "requested": "[7.0.2, )", + "resolved": "7.0.2", + "contentHash": "sFfqn4T6Ux4AbGnhS+BZvf9iVeP6b9p9bwaMT8nsefVcYH2tC9MHj3AoY+GG0nnBPmFawRqdgud08cBjBdPByQ==", + "dependencies": { + "Microsoft.CSharp": "4.7.0" + } + }, "Microsoft.Extensions.Logging": { "type": "Direct", "requested": "[6.0.0, )", diff --git a/util/Setup/Program.cs b/util/Setup/Program.cs index e6eae0749..93304c6bb 100644 --- a/util/Setup/Program.cs +++ b/util/Setup/Program.cs @@ -190,7 +190,12 @@ public class Program var vaultConnectionString = Helpers.GetValueFromEnvFile("global", "globalSettings__sqlServer__connectionString"); var migrator = new DbMigrator(vaultConnectionString, null); - migrator.MigrateMsSqlDatabaseWithRetries(false); + + var log = false; + + migrator.MigrateMsSqlDatabaseWithRetries(log); + + migrator.MigrateMsSqlDatabaseWithRetries(log, true, MigratorConstants.TransitionMigrationsFolderName); } private static bool ValidateInstallation()