Upstream/bitwarden-server
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2024-11-22 12:15:36 +01:00
Code Issues Projects Releases Wiki Activity

[BEEEP] Use MsSqlMigratorUtility for local development databases (#3850)

Browse Source 
* Update migrate.ps1 to use MsSqlMigratorUtility for dev databases
* Remove old handwritten scripts
* Migrate existing migration records
* Update Github Workflow to call MsSqlMigratorUtility directly
This commit is contained in:
Thomas Rittson 2024-03-13 09:25:20 +10:00 committed by GitHub
parent dd21d8fcf4
commit 386ff744ef
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 66 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/workflows/test-database.yml vendored
View File

@ -57,9 +57,9 @@ jobs:
run: sleep 15s run: sleep 15s
- name: Migrate SQL Server - name: Migrate SQL Server
working-directory: "dev" run: 'dotnet run --project util/MsSqlMigratorUtility/ "$CONN_STR"'
run: "./migrate.ps1" env:
shell: pwsh CONN_STR: "Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;"
- name: Migrate MySQL - name: Migrate MySQL
working-directory: "util/MySqlMigrations" working-directory: "util/MySqlMigrations"
@ -147,9 +147,9 @@ jobs:
shell: pwsh shell: pwsh
- name: Migrate - name: Migrate
working-directory: "dev" run: 'dotnet run --project util/MsSqlMigratorUtility/ "$CONN_STR"'
run: "./migrate.ps1" env:
shell: pwsh CONN_STR: "Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;"
- name: Diff .sqlproj to migrations - name: Diff .sqlproj to migrations
run: /usr/local/sqlpackage/sqlpackage /action:DeployReport /SourceFile:"Sql.dacpac" /TargetConnectionString:"Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;" /OutputPath:"report.xml" /p:IgnoreColumnOrder=True /p:IgnoreComments=True run: /usr/local/sqlpackage/sqlpackage /action:DeployReport /SourceFile:"Sql.dacpac" /TargetConnectionString:"Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;" /OutputPath:"report.xml" /p:IgnoreColumnOrder=True /p:IgnoreComments=True

75
dev/helpers/mssql/migrate_migrations.sh
View File

@ -1,12 +1,12 @@
#!/bin/bash #!/bin/bash
#
# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294) # !!! UPDATED 2024 for MsSqlMigratorUtility !!!
#
# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294)
# where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost. # where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost.
# The best workaround seems to be adding tiny delay like so: # The best workaround seems to be adding tiny delay like so:
sleep 0.1; sleep 0.1;
MIGRATE_DIRECTORY="/mnt/migrator/DbScripts"
LAST_MIGRATION_FILE="/mnt/data/last_migration"
SERVER='mssql' SERVER='mssql'
DATABASE="vault_dev" DATABASE="vault_dev"
USER="SA" USER="SA"
@ -16,58 +16,33 @@ while getopts "s" arg; do
case $arg in case $arg in
s) s)
echo "Running for self-host environment" echo "Running for self-host environment"
LAST_MIGRATION_FILE="/mnt/data/last_self_host_migration"
DATABASE="vault_dev_self_host" DATABASE="vault_dev_self_host"
;; ;;
esac esac
done done
if [ ! -f "$LAST_MIGRATION_FILE" ]; then QUERY="IF OBJECT_ID('[$DATABASE].[dbo].[Migration]') IS NULL AND OBJECT_ID('[migrations_$DATABASE].[dbo].[migrations]') IS NOT NULL
echo "No migration file, nothing to migrate to a database store"
exit 1
else
LAST_MIGRATION=$(cat $LAST_MIGRATION_FILE)
rm $LAST_MIGRATION_FILE
fi
[ -z "$LAST_MIGRATION" ]
PERFORM_MIGRATION=$?
# Create database if it does not already exist
QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE')
BEGIN BEGIN
CREATE DATABASE migrations_$DATABASE; -- Create [database].dbo.Migration with the schema expected by MsSqlMigratorUtility
END; SET ANSI_NULLS ON;
SET QUOTED_IDENTIFIER ON;
CREATE TABLE [$DATABASE].[dbo].[Migration](
[Id] [int] IDENTITY(1,1) NOT NULL,
[ScriptName] [nvarchar](255) NOT NULL,
[Applied] [datetime] NOT NULL
) ON [PRIMARY];
ALTER TABLE [$DATABASE].[dbo].[Migration] ADD CONSTRAINT [PK_Migration_Id] PRIMARY KEY CLUSTERED
(
[Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY];
-- Copy across old data
INSERT INTO [$DATABASE].[dbo].[Migration] (ScriptName, Applied)
SELECT CONCAT('Bit.Migrator.DbScripts.', [Filename]), CreationDate
FROM [migrations_$DATABASE].[dbo].[migrations];
END
" "
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY" /opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
QUERY="IF OBJECT_ID('[dbo].[migrations_$DATABASE]') IS NULL
BEGIN
CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] (
[Id] INT IDENTITY(1,1) PRIMARY KEY,
[Filename] NVARCHAR(MAX) NOT NULL,
[CreationDate] DATETIME2 (7) NULL,
);
END;"
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
record_migration () {
echo "recording $1"
local file=$(basename $1)
echo $file
local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())"
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query"
}
for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do
if (( PERFORM_MIGRATION == 0 )); then
echo "Still need to migrate $f"
else
record_migration $f
if [ "$LAST_MIGRATION" == "$f" ]; then
PERFORM_MIGRATION=0
fi
fi
done;

94
dev/helpers/mssql/run_migrations.sh
View File

@ -1,94 +0,0 @@
#!/bin/bash
# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294)
# where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost.
# The best workaround seems to be adding tiny delay like so:
sleep 0.1;
MIGRATE_DIRECTORY="/mnt/migrator/DbScripts"
SERVER='mssql'
DATABASE="vault_dev"
USER="SA"
PASSWD=$MSSQL_PASSWORD
while getopts "sp" arg; do
case $arg in
s)
echo "Running for self-host environment"
DATABASE="vault_dev_self_host"
;;
p)
echo "Running for pipeline"
MIGRATE_DIRECTORY=$MSSQL_MIGRATIONS_DIRECTORY
SERVER=$MSSQL_HOST
DATABASE=$MSSQL_DATABASE
USER=$MSSQL_USER
PASSWD=$MSSQL_PASS
esac
done
# Create databases if they do not already exist
QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = '$DATABASE')
BEGIN
CREATE DATABASE $DATABASE;
END;
GO
IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE')
BEGIN
CREATE DATABASE migrations_$DATABASE;
END;
GO
"
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
echo "Return code: $?"
# Create migrations table if it does not already exist
QUERY="IF OBJECT_ID('[migrations_$DATABASE].[dbo].[migrations]') IS NULL
BEGIN
CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] (
[Id] INT IDENTITY(1,1) PRIMARY KEY,
[Filename] NVARCHAR(MAX) NOT NULL,
[CreationDate] DATETIME2 (7) NULL,
);
END;
GO
"
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$QUERY"
echo "Return code: $?"
should_migrate () {
local file=$(basename $1)
local query="SELECT * FROM [migrations] WHERE [Filename] = '$file'"
local result=$(/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query")
if [[ "$result" =~ .*"$file".* ]]; then
return 1;
else
return 0;
fi
}
record_migration () {
echo "recording $1"
local file=$(basename $1)
echo $file
local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())"
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query"
}
migrate () {
local file=$1
echo "Performing $file"
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d $DATABASE -U $USER -P $PASSWD -I -i $file
}
for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do
BASENAME=$(basename $f)
if should_migrate $f == 1 ; then
migrate $f
record_migration $f
else
echo "Skipping $f, $BASENAME"
fi
done;

53
dev/migrate.ps1
View File

@ -2,20 +2,20 @@
# Creates the vault_dev database, and runs all the migrations. # Creates the vault_dev database, and runs all the migrations.
# Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use # Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use
# the mssql-tools container which runs under x86_64. We should monitor this # the mssql-tools container which runs under x86_64.
# in the future and investigate if we can migrate back.
# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args
param( param(
[switch]$all = $false, [switch]$all,
[switch]$postgres = $false, [switch]$postgres,
[switch]$mysql = $false, [switch]$mysql,
[switch]$mssql = $false, [switch]$mssql,
[switch]$sqlite = $false, [switch]$sqlite,
[switch]$selfhost = $false, [switch]$selfhost
[switch]$pipeline = $false
) )
# Abort on any error
$ErrorActionPreference = "Stop"
if (!$all -and !$postgres -and !$mysql -and !$sqlite) { if (!$all -and !$postgres -and !$mysql -and !$sqlite) {
$mssql = $true; $mssql = $true;
} }
@ -29,22 +29,27 @@ if ($all -or $postgres -or $mysql -or $sqlite) {
} }
if ($all -or $mssql) { if ($all -or $mssql) {
if ($selfhost) { function Get-UserSecrets {
$migrationArgs = "-s" return dotnet user-secrets list --json --project ../src/Api | ConvertFrom-Json
} elseif ($pipeline) {
$migrationArgs = "-p"
} }
Write-Host "Starting Microsoft SQL Server Migrations" if ($selfhost) {
docker run ` $msSqlConnectionString = $(Get-UserSecrets).'dev:selfHostOverride:globalSettings:sqlServer:connectionString'
-v "$(pwd)/helpers/mssql:/mnt/helpers" ` $envName = "self-host"
-v "$(pwd)/../util/Migrator:/mnt/migrator/" `
-v "$(pwd)/.data/mssql:/mnt/data" ` Write-Output "Migrating your migrations to use MsSqlMigratorUtility (if needed)"
--env-file .env ` ./migrate_migration_record.ps1 -s
--network=bitwardenserver_default ` } else {
--rm ` $msSqlConnectionString = $(Get-UserSecrets).'globalSettings:sqlServer:connectionString'
mcr.microsoft.com/mssql-tools ` $envName = "cloud"
/mnt/helpers/run_migrations.sh $migrationArgs
Write-Output "Migrating your migrations to use MsSqlMigratorUtility (if needed)"
./migrate_migration_record.ps1
}
Write-Host "Starting Microsoft SQL Server Migrations for $envName"
dotnet run --project ../util/MsSqlMigratorUtility/ "$msSqlConnectionString"
} }
$currentDir = Get-Location $currentDir = Get-Location

14
dev/migrate_migration_record.ps1
View File

@ -1,15 +1,13 @@
#!/usr/bin/env pwsh #!/usr/bin/env pwsh
# This script need only be run once # !!! UPDATED 2024 for MsSqlMigratorUtility !!!
# #
# This is a migration script for updating recording the last migration run # This is a migration script to move data from [migrations_vault_dev].[dbo].[migrations] (used by our custom
# in a file to recording migrations in a database table. It will create a # migrator script) to [vault_dev].[dbo].[Migration] (used by MsSqlMigratorUtility). It is safe to run multiple
# migrations_vault table and store all of the previously run migrations as # times because it will not perform any migration if it detects that the new table is already present.
# indicated by a last_migrations file. It will then delete this file. # This will be deleted after a few months after everyone has (presumably) migrated to the new schema.
# Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use # Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use
# the mssql-tools container which runs under x86_64. We should monitor this # the mssql-tools container which runs under x86_64.
# in the future and investigate if we can migrate back.
# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args
docker run ` docker run `
-v "$(pwd)/helpers/mssql:/mnt/helpers" ` -v "$(pwd)/helpers/mssql:/mnt/helpers" `