Store migrations in a migrations db (#1992)

* Store migrations in a migrations db

Added the -p flag to run_migrations for pipeline to run from
environment variables

Created meta-migrations script to pre-populate migrations_vault_dev
tables with run migrations for those already using the last_migrations
file

* Update dev/helpers/mssql/migrate_migrations.sh

Co-authored-by: Vince Grassia <593223+vgrassia@users.noreply.github.com>

* Update dev/helpers/mssql/run_migrations.sh

Co-authored-by: Vince Grassia <593223+vgrassia@users.noreply.github.com>

* Remove last_migration file creation

Co-authored-by: Vince Grassia <593223+vgrassia@users.noreply.github.com>

dev/helpers/mssql/migrate_migrations.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294) 
+# where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost.
+# The best workaround seems to be adding tiny delay like so:
+sleep 0.1;
+
+MIGRATE_DIRECTORY="/mnt/migrator/DbScripts"
+LAST_MIGRATION_FILE="/mnt/data/last_migration"
+SERVER='mssql'
+DATABASE="vault_dev"
+USER="SA"
+PASSWD=$MSSQL_PASSWORD
+
+while getopts "s" arg; do
+  case $arg in
+    s)
+      echo "Running for self-host environment"
+      LAST_MIGRATION_FILE="/mnt/data/last_self_host_migration"
+      DATABASE="vault_dev_self_host"
+      ;;
+  esac
+done
+
+if [ ! -f "$LAST_MIGRATION_FILE" ]; then
+  echo "No migration file, nothing to migrate to a database store"
+  exit 1
+else
+  LAST_MIGRATION=$(cat $LAST_MIGRATION_FILE)
+  rm $LAST_MIGRATION_FILE
+fi
+
+[ -z "$LAST_MIGRATION" ]
+PERFORM_MIGRATION=$?
+
+# Create database if it does not already exist
+QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE')
+BEGIN
+  CREATE DATABASE migrations_$DATABASE;
+END;
+"
+
+/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
+
+QUERY="IF OBJECT_ID('[dbo].[migrations_$DATABASE]') IS NULL
+BEGIN
+  CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] (
+      [Id]                   INT IDENTITY(1,1) PRIMARY KEY,
+      [Filename]             NVARCHAR(MAX) NOT NULL,
+      [CreationDate]         DATETIME2 (7)    NULL,
+  );
+END;"
+
+/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
+
+record_migration () {
+  echo "recording $1"
+  local file=$(basename $1)
+  echo $file
+  local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())"
+  /opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query"
+}
+
+for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do
+  if (( PERFORM_MIGRATION == 0 )); then
+    echo "Still need to migrate $f"
+  else
+    record_migration $f
+    if [ "$LAST_MIGRATION" == "$f" ]; then
+      PERFORM_MIGRATION=0
+    fi
+  fi
+done;

dev/helpers/mssql/run_migrations.sh

@@ -6,73 +6,76 @@
 sleep 0.1;
 
 MIGRATE_DIRECTORY="/mnt/migrator/DbScripts"
-LAST_MIGRATION_FILE="/mnt/data/last_migration"
 SERVER='mssql'
 DATABASE="vault_dev"
 USER="SA"
 PASSWD=$MSSQL_PASSWORD
 
-while getopts "rs" arg; do
+while getopts "sp" arg; do
   case $arg in
-    r)
-      echo "Rerunning the last migration"
-      RERUN=1
-      ;;
     s)
       echo "Running for self-host environment"
-      LAST_MIGRATION_FILE="/mnt/data/last_self_host_migration"
       DATABASE="vault_dev_self_host"
       ;;
+    p)
+      echo "Running for pipeline"
+      MIGRATE_DIRECTORY=$MSSQL_MIGRATIONS_DIRECTORY
+      SERVER=$MSSQL_HOST
+      DATABASE=$MSSQL_DATABASE
+      USER=$MSSQL_USER
+      PASSWD=$MSSQL_PASS
   esac
 done
 
-if [ ! -f "$LAST_MIGRATION_FILE" ]; then
+# Create databases if they do not already exist
-  echo "$LAST_MIGRATION_FILE not found!"
-  echo "This will run all migrations which might cause unexpected behaviour if the database is not empty."
-  echo
-  read -p "Run all Migrations? (y/N) " -n 1 -r
-  echo
-  if [[ ! $REPLY =~ ^[Yy]$ ]]
-  then
-      exit 1
-  fi
-  LAST_MIGRATION=""
-else
-  LAST_MIGRATION=$(cat $LAST_MIGRATION_FILE)
-fi
-
-[ -z "$LAST_MIGRATION" ]
-PERFORM_MIGRATION=$?
-
-
-# Create database if it does not already exist
 QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = '$DATABASE')
 BEGIN
   CREATE DATABASE $DATABASE;
+END;
+
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE')
+BEGIN
+  CREATE DATABASE migrations_$DATABASE;
+  CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] (
+      [Id]                   INT IDENTITY(1,1) PRIMARY KEY,
+      [Filename]             NVARCHAR(MAX) NOT NULL,
+      [CreationDate]         DATETIME2 (7)    NULL,
+  );
 END;"
 
 /opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
 
+should_migrate () {
+  local file=$(basename $1)
+  local query="SELECT * FROM [migrations] WHERE [Filename] = '$file'"
+  local result=$(/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query")
+  if [[ "$result" =~ .*"$file".* ]]; then
+    return 1;
+  else
+    return 0;
+  fi
+}
+
+record_migration () {
+  echo "recording $1"
+  local file=$(basename $1)
+  echo $file
+  local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())"
+  /opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query"
+}
+
 migrate () {
   local file=$1
   echo "Performing $file"
   /opt/mssql-tools/bin/sqlcmd -S $SERVER -d $DATABASE -U $USER -P $PASSWD -I -i $file
-  echo $file > $LAST_MIGRATION_FILE
 }
 
 for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do
-  if (( PERFORM_MIGRATION == 0 )); then
+  BASENAME=$(basename $f)
+  if should_migrate $f == 1 ; then
     migrate $f
+    record_migration $f
   else
-    echo "Skipping $f"
+    echo "Skipping $f, $BASENAME"
-    if [ "$LAST_MIGRATION" == "$f" ]; then
-      PERFORM_MIGRATION=0
-
-      # Rerun last migration
-      if [ -n "$RERUN" ]; then
-        migrate $f
-        unset RERUN
-      fi
-    fi
   fi
 done;

dev/migrate_migration_record.ps1

@@ -0,0 +1,23 @@
+#!/usr/bin/env pwsh
+# This script need only be run once
+#
+# This is a migration script for updating recording the last migration run
+# in a file to recording migrations in a database table. It will create a
+# migrations_vault table and store all of the previously run migrations as
+# indicated by a last_migrations file. It will then delete this file.
+
+# Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use
+#  the mssql-tools container which runs under x86_64. We should monitor this
+#  in the future and investigate if we can migrate back.
+# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args
+
+docker run `
+  -v "$(pwd)/helpers/mssql:/mnt/helpers" `
+  -v "$(pwd)/../util/Migrator:/mnt/migrator/" `
+  -v "$(pwd)/.data/mssql:/mnt/data" `
+  --env-file .env `
+  --network=bitwardenserver_default `
+  --rm `
+  -it `
+  mcr.microsoft.com/mssql-tools `
+  /mnt/helpers/migrate_migrations.sh @args