diff --git a/.github/workflows/test-database.yml b/.github/workflows/test-database.yml index cf62a1f43..b57cc8786 100644 --- a/.github/workflows/test-database.yml +++ b/.github/workflows/test-database.yml @@ -57,9 +57,9 @@ jobs: run: sleep 15s - name: Migrate SQL Server - working-directory: "dev" - run: "./migrate.ps1" - shell: pwsh + run: 'dotnet run --project util/MsSqlMigratorUtility/ "$CONN_STR"' + env: + CONN_STR: "Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;" - name: Migrate MySQL working-directory: "util/MySqlMigrations" @@ -147,9 +147,9 @@ jobs: shell: pwsh - name: Migrate - working-directory: "dev" - run: "./migrate.ps1" - shell: pwsh + run: 'dotnet run --project util/MsSqlMigratorUtility/ "$CONN_STR"' + env: + CONN_STR: "Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;" - name: Diff .sqlproj to migrations run: /usr/local/sqlpackage/sqlpackage /action:DeployReport /SourceFile:"Sql.dacpac" /TargetConnectionString:"Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;" /OutputPath:"report.xml" /p:IgnoreColumnOrder=True /p:IgnoreComments=True diff --git a/dev/helpers/mssql/migrate_migrations.sh b/dev/helpers/mssql/migrate_migrations.sh index 56f8da555..f8993bc14 100755 --- a/dev/helpers/mssql/migrate_migrations.sh +++ b/dev/helpers/mssql/migrate_migrations.sh @@ -1,12 +1,12 @@ #!/bin/bash - -# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294) +# +# !!! UPDATED 2024 for MsSqlMigratorUtility !!! +# +# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294) # where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost. # The best workaround seems to be adding tiny delay like so: sleep 0.1; -MIGRATE_DIRECTORY="/mnt/migrator/DbScripts" -LAST_MIGRATION_FILE="/mnt/data/last_migration" SERVER='mssql' DATABASE="vault_dev" USER="SA" @@ -16,58 +16,33 @@ while getopts "s" arg; do case $arg in s) echo "Running for self-host environment" - LAST_MIGRATION_FILE="/mnt/data/last_self_host_migration" DATABASE="vault_dev_self_host" ;; esac done -if [ ! -f "$LAST_MIGRATION_FILE" ]; then - echo "No migration file, nothing to migrate to a database store" - exit 1 -else - LAST_MIGRATION=$(cat $LAST_MIGRATION_FILE) - rm $LAST_MIGRATION_FILE -fi - -[ -z "$LAST_MIGRATION" ] -PERFORM_MIGRATION=$? - -# Create database if it does not already exist -QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE') +QUERY="IF OBJECT_ID('[$DATABASE].[dbo].[Migration]') IS NULL AND OBJECT_ID('[migrations_$DATABASE].[dbo].[migrations]') IS NOT NULL BEGIN - CREATE DATABASE migrations_$DATABASE; -END; + -- Create [database].dbo.Migration with the schema expected by MsSqlMigratorUtility + SET ANSI_NULLS ON; + SET QUOTED_IDENTIFIER ON; + + CREATE TABLE [$DATABASE].[dbo].[Migration]( + [Id] [int] IDENTITY(1,1) NOT NULL, + [ScriptName] [nvarchar](255) NOT NULL, + [Applied] [datetime] NOT NULL + ) ON [PRIMARY]; + + ALTER TABLE [$DATABASE].[dbo].[Migration] ADD CONSTRAINT [PK_Migration_Id] PRIMARY KEY CLUSTERED + ( + [Id] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]; + + -- Copy across old data + INSERT INTO [$DATABASE].[dbo].[Migration] (ScriptName, Applied) + SELECT CONCAT('Bit.Migrator.DbScripts.', [Filename]), CreationDate + FROM [migrations_$DATABASE].[dbo].[migrations]; +END " /opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY" - -QUERY="IF OBJECT_ID('[dbo].[migrations_$DATABASE]') IS NULL -BEGIN - CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] ( - [Id] INT IDENTITY(1,1) PRIMARY KEY, - [Filename] NVARCHAR(MAX) NOT NULL, - [CreationDate] DATETIME2 (7) NULL, - ); -END;" - -/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY" - -record_migration () { - echo "recording $1" - local file=$(basename $1) - echo $file - local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())" - /opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query" -} - -for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do - if (( PERFORM_MIGRATION == 0 )); then - echo "Still need to migrate $f" - else - record_migration $f - if [ "$LAST_MIGRATION" == "$f" ]; then - PERFORM_MIGRATION=0 - fi - fi -done; diff --git a/dev/helpers/mssql/run_migrations.sh b/dev/helpers/mssql/run_migrations.sh deleted file mode 100755 index 2ff75e7c5..000000000 --- a/dev/helpers/mssql/run_migrations.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash - -# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294) -# where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost. -# The best workaround seems to be adding tiny delay like so: -sleep 0.1; - -MIGRATE_DIRECTORY="/mnt/migrator/DbScripts" -SERVER='mssql' -DATABASE="vault_dev" -USER="SA" -PASSWD=$MSSQL_PASSWORD - -while getopts "sp" arg; do - case $arg in - s) - echo "Running for self-host environment" - DATABASE="vault_dev_self_host" - ;; - p) - echo "Running for pipeline" - MIGRATE_DIRECTORY=$MSSQL_MIGRATIONS_DIRECTORY - SERVER=$MSSQL_HOST - DATABASE=$MSSQL_DATABASE - USER=$MSSQL_USER - PASSWD=$MSSQL_PASS - esac -done - -# Create databases if they do not already exist -QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = '$DATABASE') -BEGIN - CREATE DATABASE $DATABASE; -END; - -GO -IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE') -BEGIN - CREATE DATABASE migrations_$DATABASE; -END; - -GO -" -/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY" -echo "Return code: $?" - -# Create migrations table if it does not already exist -QUERY="IF OBJECT_ID('[migrations_$DATABASE].[dbo].[migrations]') IS NULL -BEGIN - CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] ( - [Id] INT IDENTITY(1,1) PRIMARY KEY, - [Filename] NVARCHAR(MAX) NOT NULL, - [CreationDate] DATETIME2 (7) NULL, - ); -END; -GO -" -/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$QUERY" -echo "Return code: $?" - -should_migrate () { - local file=$(basename $1) - local query="SELECT * FROM [migrations] WHERE [Filename] = '$file'" - local result=$(/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query") - if [[ "$result" =~ .*"$file".* ]]; then - return 1; - else - return 0; - fi -} - -record_migration () { - echo "recording $1" - local file=$(basename $1) - echo $file - local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())" - /opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query" -} - -migrate () { - local file=$1 - echo "Performing $file" - /opt/mssql-tools/bin/sqlcmd -S $SERVER -d $DATABASE -U $USER -P $PASSWD -I -i $file -} - -for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do - BASENAME=$(basename $f) - if should_migrate $f == 1 ; then - migrate $f - record_migration $f - else - echo "Skipping $f, $BASENAME" - fi -done; diff --git a/dev/migrate.ps1 b/dev/migrate.ps1 index c1894342b..9aec956db 100755 --- a/dev/migrate.ps1 +++ b/dev/migrate.ps1 @@ -2,20 +2,20 @@ # Creates the vault_dev database, and runs all the migrations. # Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use -# the mssql-tools container which runs under x86_64. We should monitor this -# in the future and investigate if we can migrate back. -# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args +# the mssql-tools container which runs under x86_64. param( - [switch]$all = $false, - [switch]$postgres = $false, - [switch]$mysql = $false, - [switch]$mssql = $false, - [switch]$sqlite = $false, - [switch]$selfhost = $false, - [switch]$pipeline = $false + [switch]$all, + [switch]$postgres, + [switch]$mysql, + [switch]$mssql, + [switch]$sqlite, + [switch]$selfhost ) +# Abort on any error +$ErrorActionPreference = "Stop" + if (!$all -and !$postgres -and !$mysql -and !$sqlite) { $mssql = $true; } @@ -29,22 +29,27 @@ if ($all -or $postgres -or $mysql -or $sqlite) { } if ($all -or $mssql) { - if ($selfhost) { - $migrationArgs = "-s" - } elseif ($pipeline) { - $migrationArgs = "-p" + function Get-UserSecrets { + return dotnet user-secrets list --json --project ../src/Api | ConvertFrom-Json } - Write-Host "Starting Microsoft SQL Server Migrations" - docker run ` - -v "$(pwd)/helpers/mssql:/mnt/helpers" ` - -v "$(pwd)/../util/Migrator:/mnt/migrator/" ` - -v "$(pwd)/.data/mssql:/mnt/data" ` - --env-file .env ` - --network=bitwardenserver_default ` - --rm ` - mcr.microsoft.com/mssql-tools ` - /mnt/helpers/run_migrations.sh $migrationArgs + if ($selfhost) { + $msSqlConnectionString = $(Get-UserSecrets).'dev:selfHostOverride:globalSettings:sqlServer:connectionString' + $envName = "self-host" + + Write-Output "Migrating your migrations to use MsSqlMigratorUtility (if needed)" + ./migrate_migration_record.ps1 -s + } else { + $msSqlConnectionString = $(Get-UserSecrets).'globalSettings:sqlServer:connectionString' + $envName = "cloud" + + Write-Output "Migrating your migrations to use MsSqlMigratorUtility (if needed)" + ./migrate_migration_record.ps1 + } + + Write-Host "Starting Microsoft SQL Server Migrations for $envName" + + dotnet run --project ../util/MsSqlMigratorUtility/ "$msSqlConnectionString" } $currentDir = Get-Location diff --git a/dev/migrate_migration_record.ps1 b/dev/migrate_migration_record.ps1 index 7ec8f89b3..17521edf9 100755 --- a/dev/migrate_migration_record.ps1 +++ b/dev/migrate_migration_record.ps1 @@ -1,15 +1,13 @@ #!/usr/bin/env pwsh -# This script need only be run once +# !!! UPDATED 2024 for MsSqlMigratorUtility !!! # -# This is a migration script for updating recording the last migration run -# in a file to recording migrations in a database table. It will create a -# migrations_vault table and store all of the previously run migrations as -# indicated by a last_migrations file. It will then delete this file. +# This is a migration script to move data from [migrations_vault_dev].[dbo].[migrations] (used by our custom +# migrator script) to [vault_dev].[dbo].[Migration] (used by MsSqlMigratorUtility). It is safe to run multiple +# times because it will not perform any migration if it detects that the new table is already present. +# This will be deleted after a few months after everyone has (presumably) migrated to the new schema. # Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use -# the mssql-tools container which runs under x86_64. We should monitor this -# in the future and investigate if we can migrate back. -# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args +# the mssql-tools container which runs under x86_64. docker run ` -v "$(pwd)/helpers/mssql:/mnt/helpers" `