diff --git a/dev/helpers/mssql/migrate_migrations.sh b/dev/helpers/mssql/migrate_migrations.sh new file mode 100755 index 000000000..56f8da555 --- /dev/null +++ b/dev/helpers/mssql/migrate_migrations.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294) +# where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost. +# The best workaround seems to be adding tiny delay like so: +sleep 0.1; + +MIGRATE_DIRECTORY="/mnt/migrator/DbScripts" +LAST_MIGRATION_FILE="/mnt/data/last_migration" +SERVER='mssql' +DATABASE="vault_dev" +USER="SA" +PASSWD=$MSSQL_PASSWORD + +while getopts "s" arg; do + case $arg in + s) + echo "Running for self-host environment" + LAST_MIGRATION_FILE="/mnt/data/last_self_host_migration" + DATABASE="vault_dev_self_host" + ;; + esac +done + +if [ ! -f "$LAST_MIGRATION_FILE" ]; then + echo "No migration file, nothing to migrate to a database store" + exit 1 +else + LAST_MIGRATION=$(cat $LAST_MIGRATION_FILE) + rm $LAST_MIGRATION_FILE +fi + +[ -z "$LAST_MIGRATION" ] +PERFORM_MIGRATION=$? + +# Create database if it does not already exist +QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE') +BEGIN + CREATE DATABASE migrations_$DATABASE; +END; +" + +/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY" + +QUERY="IF OBJECT_ID('[dbo].[migrations_$DATABASE]') IS NULL +BEGIN + CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] ( + [Id] INT IDENTITY(1,1) PRIMARY KEY, + [Filename] NVARCHAR(MAX) NOT NULL, + [CreationDate] DATETIME2 (7) NULL, + ); +END;" + +/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY" + +record_migration () { + echo "recording $1" + local file=$(basename $1) + echo $file + local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())" + /opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query" +} + +for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do + if (( PERFORM_MIGRATION == 0 )); then + echo "Still need to migrate $f" + else + record_migration $f + if [ "$LAST_MIGRATION" == "$f" ]; then + PERFORM_MIGRATION=0 + fi + fi +done; diff --git a/dev/helpers/mssql/run_migrations.sh b/dev/helpers/mssql/run_migrations.sh index 4024b1695..6710410ce 100755 --- a/dev/helpers/mssql/run_migrations.sh +++ b/dev/helpers/mssql/run_migrations.sh @@ -6,73 +6,76 @@ sleep 0.1; MIGRATE_DIRECTORY="/mnt/migrator/DbScripts" -LAST_MIGRATION_FILE="/mnt/data/last_migration" SERVER='mssql' DATABASE="vault_dev" USER="SA" PASSWD=$MSSQL_PASSWORD -while getopts "rs" arg; do +while getopts "sp" arg; do case $arg in - r) - echo "Rerunning the last migration" - RERUN=1 - ;; s) echo "Running for self-host environment" - LAST_MIGRATION_FILE="/mnt/data/last_self_host_migration" DATABASE="vault_dev_self_host" ;; + p) + echo "Running for pipeline" + MIGRATE_DIRECTORY=$MSSQL_MIGRATIONS_DIRECTORY + SERVER=$MSSQL_HOST + DATABASE=$MSSQL_DATABASE + USER=$MSSQL_USER + PASSWD=$MSSQL_PASS esac done -if [ ! -f "$LAST_MIGRATION_FILE" ]; then - echo "$LAST_MIGRATION_FILE not found!" - echo "This will run all migrations which might cause unexpected behaviour if the database is not empty." - echo - read -p "Run all Migrations? (y/N) " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]] - then - exit 1 - fi - LAST_MIGRATION="" -else - LAST_MIGRATION=$(cat $LAST_MIGRATION_FILE) -fi - -[ -z "$LAST_MIGRATION" ] -PERFORM_MIGRATION=$? - - -# Create database if it does not already exist +# Create databases if they do not already exist QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = '$DATABASE') BEGIN CREATE DATABASE $DATABASE; +END; + +IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE') +BEGIN + CREATE DATABASE migrations_$DATABASE; + CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] ( + [Id] INT IDENTITY(1,1) PRIMARY KEY, + [Filename] NVARCHAR(MAX) NOT NULL, + [CreationDate] DATETIME2 (7) NULL, + ); END;" /opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY" +should_migrate () { + local file=$(basename $1) + local query="SELECT * FROM [migrations] WHERE [Filename] = '$file'" + local result=$(/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query") + if [[ "$result" =~ .*"$file".* ]]; then + return 1; + else + return 0; + fi +} + +record_migration () { + echo "recording $1" + local file=$(basename $1) + echo $file + local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())" + /opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query" +} + migrate () { local file=$1 echo "Performing $file" /opt/mssql-tools/bin/sqlcmd -S $SERVER -d $DATABASE -U $USER -P $PASSWD -I -i $file - echo $file > $LAST_MIGRATION_FILE } for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do - if (( PERFORM_MIGRATION == 0 )); then + BASENAME=$(basename $f) + if should_migrate $f == 1 ; then migrate $f + record_migration $f else - echo "Skipping $f" - if [ "$LAST_MIGRATION" == "$f" ]; then - PERFORM_MIGRATION=0 - - # Rerun last migration - if [ -n "$RERUN" ]; then - migrate $f - unset RERUN - fi - fi + echo "Skipping $f, $BASENAME" fi done; diff --git a/dev/migrate_migration_record.ps1 b/dev/migrate_migration_record.ps1 new file mode 100755 index 000000000..7ec8f89b3 --- /dev/null +++ b/dev/migrate_migration_record.ps1 @@ -0,0 +1,23 @@ +#!/usr/bin/env pwsh +# This script need only be run once +# +# This is a migration script for updating recording the last migration run +# in a file to recording migrations in a database table. It will create a +# migrations_vault table and store all of the previously run migrations as +# indicated by a last_migrations file. It will then delete this file. + +# Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use +# the mssql-tools container which runs under x86_64. We should monitor this +# in the future and investigate if we can migrate back. +# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args + +docker run ` + -v "$(pwd)/helpers/mssql:/mnt/helpers" ` + -v "$(pwd)/../util/Migrator:/mnt/migrator/" ` + -v "$(pwd)/.data/mssql:/mnt/data" ` + --env-file .env ` + --network=bitwardenserver_default ` + --rm ` + -it ` + mcr.microsoft.com/mssql-tools ` + /mnt/helpers/migrate_migrations.sh @args