[BEEEP] Use MsSqlMigratorUtility for local development databases (#3850)

* Update migrate.ps1 to use MsSqlMigratorUtility for dev databases * Remove old handwritten scripts * Migrate existing migration records * Update Github Workflow to call MsSqlMigratorUtility directly
2024-11-21 12:05:42 +01:00 · 2024-03-13 09:25:20 +10:00 · 2024-03-13 09:25:20 +10:00 · 386ff744ef
commit 386ff744ef
parent dd21d8fcf4
5 changed files with 66 additions and 182 deletions
--- a/.github/workflows/test-database.yml
+++ b/.github/workflows/test-database.yml
@ -57,9 +57,9 @@ jobs:
        run: sleep 15s

      - name: Migrate SQL Server
-        working-directory: "dev"
-        run: "./migrate.ps1"
-        shell: pwsh
+        run: 'dotnet run --project util/MsSqlMigratorUtility/ "$CONN_STR"'
+        env:
+          CONN_STR: "Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;"

      - name: Migrate MySQL
        working-directory: "util/MySqlMigrations"
@ -147,9 +147,9 @@ jobs:
        shell: pwsh

      - name: Migrate
-        working-directory: "dev"
-        run: "./migrate.ps1"
-        shell: pwsh
+        run: 'dotnet run --project util/MsSqlMigratorUtility/ "$CONN_STR"'
+        env:
+          CONN_STR: "Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;"

      - name: Diff .sqlproj to migrations
        run: /usr/local/sqlpackage/sqlpackage /action:DeployReport /SourceFile:"Sql.dacpac" /TargetConnectionString:"Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;" /OutputPath:"report.xml" /p:IgnoreColumnOrder=True /p:IgnoreComments=True
--- a/dev/helpers/mssql/migrate_migrations.sh
+++ b/dev/helpers/mssql/migrate_migrations.sh
@ -1,12 +1,12 @@
 #!/bin/bash
-
-# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294) 
+#
+# !!! UPDATED 2024 for MsSqlMigratorUtility !!!
+#
+# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294)
 # where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost.
 # The best workaround seems to be adding tiny delay like so:
 sleep 0.1;

-MIGRATE_DIRECTORY="/mnt/migrator/DbScripts"
-LAST_MIGRATION_FILE="/mnt/data/last_migration"
 SERVER='mssql'
 DATABASE="vault_dev"
 USER="SA"
@ -16,58 +16,33 @@ while getopts "s" arg; do
  case $arg in
    s)
      echo "Running for self-host environment"
-      LAST_MIGRATION_FILE="/mnt/data/last_self_host_migration"
      DATABASE="vault_dev_self_host"
      ;;
  esac
 done

-if [ ! -f "$LAST_MIGRATION_FILE" ]; then
-  echo "No migration file, nothing to migrate to a database store"
-  exit 1
-else
-  LAST_MIGRATION=$(cat $LAST_MIGRATION_FILE)
-  rm $LAST_MIGRATION_FILE
-fi
-
-[ -z "$LAST_MIGRATION" ]
-PERFORM_MIGRATION=$?
-
-# Create database if it does not already exist
-QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE')
+QUERY="IF OBJECT_ID('[$DATABASE].[dbo].[Migration]') IS NULL AND OBJECT_ID('[migrations_$DATABASE].[dbo].[migrations]') IS NOT NULL
 BEGIN
-  CREATE DATABASE migrations_$DATABASE;
-END;
+    -- Create [database].dbo.Migration with the schema expected by MsSqlMigratorUtility
+    SET ANSI_NULLS ON;
+    SET QUOTED_IDENTIFIER ON;
+
+    CREATE TABLE [$DATABASE].[dbo].[Migration](
+        [Id] [int] IDENTITY(1,1) NOT NULL,
+        [ScriptName] [nvarchar](255) NOT NULL,
+        [Applied] [datetime] NOT NULL
+    ) ON [PRIMARY];
+
+    ALTER TABLE [$DATABASE].[dbo].[Migration] ADD  CONSTRAINT [PK_Migration_Id] PRIMARY KEY CLUSTERED
+    (
+        [Id] ASC
+    )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY];
+
+    -- Copy across old data
+    INSERT INTO [$DATABASE].[dbo].[Migration] (ScriptName, Applied)
+    SELECT CONCAT('Bit.Migrator.DbScripts.', [Filename]), CreationDate
+    FROM [migrations_$DATABASE].[dbo].[migrations];
+END
 "

 /opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
-
-QUERY="IF OBJECT_ID('[dbo].[migrations_$DATABASE]') IS NULL
-BEGIN
-  CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] (
-      [Id]                   INT IDENTITY(1,1) PRIMARY KEY,
-      [Filename]             NVARCHAR(MAX) NOT NULL,
-      [CreationDate]         DATETIME2 (7)    NULL,
-  );
-END;"
-
-/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
-
-record_migration () {
-  echo "recording $1"
-  local file=$(basename $1)
-  echo $file
-  local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())"
-  /opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query"
-}
-
-for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do
-  if (( PERFORM_MIGRATION == 0 )); then
-    echo "Still need to migrate $f"
-  else
-    record_migration $f
-    if [ "$LAST_MIGRATION" == "$f" ]; then
-      PERFORM_MIGRATION=0
-    fi
-  fi
-done;
--- a/dev/helpers/mssql/run_migrations.sh
+++ b/dev/helpers/mssql/run_migrations.sh
@ -1,94 +0,0 @@
-#!/bin/bash
-
-# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294) 
-# where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost.
-# The best workaround seems to be adding tiny delay like so:
-sleep 0.1;
-
-MIGRATE_DIRECTORY="/mnt/migrator/DbScripts"
-SERVER='mssql'
-DATABASE="vault_dev"
-USER="SA"
-PASSWD=$MSSQL_PASSWORD
-
-while getopts "sp" arg; do
-  case $arg in
-    s)
-      echo "Running for self-host environment"
-      DATABASE="vault_dev_self_host"
-      ;;
-    p)
-      echo "Running for pipeline"
-      MIGRATE_DIRECTORY=$MSSQL_MIGRATIONS_DIRECTORY
-      SERVER=$MSSQL_HOST
-      DATABASE=$MSSQL_DATABASE
-      USER=$MSSQL_USER
-      PASSWD=$MSSQL_PASS
-  esac
-done
-
-# Create databases if they do not already exist
-QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = '$DATABASE')
-BEGIN
-  CREATE DATABASE $DATABASE;
-END;
-
-GO
-IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE')
-BEGIN
-  CREATE DATABASE migrations_$DATABASE;
-END;
-
-GO
-"
-/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
-echo "Return code: $?"
-
-# Create migrations table if it does not already exist
-QUERY="IF OBJECT_ID('[migrations_$DATABASE].[dbo].[migrations]') IS NULL
-BEGIN
-    CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] (
-      [Id]                   INT IDENTITY(1,1) PRIMARY KEY,
-      [Filename]             NVARCHAR(MAX) NOT NULL,
-      [CreationDate]         DATETIME2 (7)    NULL,
-    );
-END;
-GO
-"
-/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$QUERY"
-echo "Return code: $?"
-
-should_migrate () {
-  local file=$(basename $1)
-  local query="SELECT * FROM [migrations] WHERE [Filename] = '$file'"
-  local result=$(/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query")
-  if [[ "$result" =~ .*"$file".* ]]; then
-    return 1;
-  else
-    return 0;
-  fi
-}
-
-record_migration () {
-  echo "recording $1"
-  local file=$(basename $1)
-  echo $file
-  local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())"
-  /opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query"
-}
-
-migrate () {
-  local file=$1
-  echo "Performing $file"
-  /opt/mssql-tools/bin/sqlcmd -S $SERVER -d $DATABASE -U $USER -P $PASSWD -I -i $file
-}
-
-for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do
-  BASENAME=$(basename $f)
-  if should_migrate $f == 1 ; then
-    migrate $f
-    record_migration $f
-  else
-    echo "Skipping $f, $BASENAME"
-  fi
-done;
--- a/dev/migrate.ps1
+++ b/dev/migrate.ps1
@ -2,20 +2,20 @@
 # Creates the vault_dev database, and runs all the migrations.

 # Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use
-#  the mssql-tools container which runs under x86_64. We should monitor this
-#  in the future and investigate if we can migrate back.
-# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args
+#  the mssql-tools container which runs under x86_64.

 param(
-  [switch]$all = $false,
-  [switch]$postgres = $false,
-  [switch]$mysql = $false,
-  [switch]$mssql = $false,
-  [switch]$sqlite = $false,
-  [switch]$selfhost = $false,
-  [switch]$pipeline = $false
+  [switch]$all,
+  [switch]$postgres,
+  [switch]$mysql,
+  [switch]$mssql,
+  [switch]$sqlite,
+  [switch]$selfhost
 )

+# Abort on any error
+$ErrorActionPreference = "Stop"
+
 if (!$all -and !$postgres -and !$mysql -and !$sqlite) {
  $mssql = $true;
 }
@ -29,22 +29,27 @@ if ($all -or $postgres -or $mysql -or $sqlite) {
 }

 if ($all -or $mssql) {
-  if ($selfhost) {
-    $migrationArgs = "-s"
-  } elseif ($pipeline) {
-    $migrationArgs = "-p"
+  function Get-UserSecrets {
+    return dotnet user-secrets list --json --project ../src/Api | ConvertFrom-Json
  }

-  Write-Host "Starting Microsoft SQL Server Migrations"
-  docker run `
-    -v "$(pwd)/helpers/mssql:/mnt/helpers" `
-    -v "$(pwd)/../util/Migrator:/mnt/migrator/" `
-    -v "$(pwd)/.data/mssql:/mnt/data" `
-    --env-file .env `
-    --network=bitwardenserver_default `
-    --rm `
-    mcr.microsoft.com/mssql-tools `
-    /mnt/helpers/run_migrations.sh $migrationArgs
+  if ($selfhost) {
+    $msSqlConnectionString = $(Get-UserSecrets).'dev:selfHostOverride:globalSettings:sqlServer:connectionString'
+    $envName = "self-host"
+
+    Write-Output "Migrating your migrations to use MsSqlMigratorUtility (if needed)"
+    ./migrate_migration_record.ps1 -s
+  } else {
+    $msSqlConnectionString = $(Get-UserSecrets).'globalSettings:sqlServer:connectionString'
+    $envName = "cloud"
+
+    Write-Output "Migrating your migrations to use MsSqlMigratorUtility (if needed)"
+    ./migrate_migration_record.ps1
+  }
+
+  Write-Host "Starting Microsoft SQL Server Migrations for $envName"
+
+  dotnet run --project ../util/MsSqlMigratorUtility/ "$msSqlConnectionString"
 }

 $currentDir = Get-Location
--- a/dev/migrate_migration_record.ps1
+++ b/dev/migrate_migration_record.ps1
@ -1,15 +1,13 @@
 #!/usr/bin/env pwsh
-# This script need only be run once
+# !!! UPDATED 2024 for MsSqlMigratorUtility !!!
 #
-# This is a migration script for updating recording the last migration run
-# in a file to recording migrations in a database table. It will create a
-# migrations_vault table and store all of the previously run migrations as
-# indicated by a last_migrations file. It will then delete this file.
+# This is a migration script to move data from [migrations_vault_dev].[dbo].[migrations] (used by our custom
+# migrator script) to [vault_dev].[dbo].[Migration] (used by MsSqlMigratorUtility). It is safe to run multiple
+# times because it will not perform any migration if it detects that the new table is already present.
+# This will be deleted after a few months after everyone has (presumably) migrated to the new schema.

 # Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use
-#  the mssql-tools container which runs under x86_64. We should monitor this
-#  in the future and investigate if we can migrate back.
-# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args
+# the mssql-tools container which runs under x86_64.

 docker run `
  -v "$(pwd)/helpers/mssql:/mnt/helpers" `