mirror of
https://github.com/bitwarden/server.git
synced 2024-11-21 12:05:42 +01:00
Store migrations in a migrations db (#1992)
* Store migrations in a migrations db Added the -p flag to run_migrations for pipeline to run from environment variables Created meta-migrations script to pre-populate migrations_vault_dev tables with run migrations for those already using the last_migrations file * Update dev/helpers/mssql/migrate_migrations.sh Co-authored-by: Vince Grassia <593223+vgrassia@users.noreply.github.com> * Update dev/helpers/mssql/run_migrations.sh Co-authored-by: Vince Grassia <593223+vgrassia@users.noreply.github.com> * Remove last_migration file creation Co-authored-by: Vince Grassia <593223+vgrassia@users.noreply.github.com>
This commit is contained in:
Matt Gibson
GitHub
parent
53241f16e0
commit
47b1e5317c
73
dev/helpers/mssql/migrate_migrations.sh
Executable file
73
dev/helpers/mssql/migrate_migrations.sh
Executable file
@ -0,0 +1,73 @@
|
||||
#!/bin/bash
|
||||
|
||||
# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294)
|
||||
# where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost.
|
||||
# The best workaround seems to be adding tiny delay like so:
|
||||
sleep 0.1;
|
||||
|
||||
MIGRATE_DIRECTORY="/mnt/migrator/DbScripts"
|
||||
LAST_MIGRATION_FILE="/mnt/data/last_migration"
|
||||
SERVER='mssql'
|
||||
DATABASE="vault_dev"
|
||||
USER="SA"
|
||||
PASSWD=$MSSQL_PASSWORD
|
||||
|
||||
while getopts "s" arg; do
|
||||
case $arg in
|
||||
s)
|
||||
echo "Running for self-host environment"
|
||||
LAST_MIGRATION_FILE="/mnt/data/last_self_host_migration"
|
||||
DATABASE="vault_dev_self_host"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -f "$LAST_MIGRATION_FILE" ]; then
|
||||
echo "No migration file, nothing to migrate to a database store"
|
||||
exit 1
|
||||
else
|
||||
LAST_MIGRATION=$(cat $LAST_MIGRATION_FILE)
|
||||
rm $LAST_MIGRATION_FILE
|
||||
fi
|
||||
|
||||
[ -z "$LAST_MIGRATION" ]
|
||||
PERFORM_MIGRATION=$?
|
||||
|
||||
# Create database if it does not already exist
|
||||
QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE')
|
||||
BEGIN
|
||||
CREATE DATABASE migrations_$DATABASE;
|
||||
END;
|
||||
"
|
||||
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
|
||||
|
||||
QUERY="IF OBJECT_ID('[dbo].[migrations_$DATABASE]') IS NULL
|
||||
BEGIN
|
||||
CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] (
|
||||
[Id] INT IDENTITY(1,1) PRIMARY KEY,
|
||||
[Filename] NVARCHAR(MAX) NOT NULL,
|
||||
[CreationDate] DATETIME2 (7) NULL,
|
||||
);
|
||||
END;"
|
||||
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
|
||||
|
||||
record_migration () {
|
||||
echo "recording $1"
|
||||
local file=$(basename $1)
|
||||
echo $file
|
||||
local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())"
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query"
|
||||
}
|
||||
|
||||
for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do
|
||||
if (( PERFORM_MIGRATION == 0 )); then
|
||||
echo "Still need to migrate $f"
|
||||
else
|
||||
record_migration $f
|
||||
if [ "$LAST_MIGRATION" == "$f" ]; then
|
||||
PERFORM_MIGRATION=0
|
||||
fi
|
||||
fi
|
||||
done;
|
||||
@ -6,73 +6,76 @@
|
||||
sleep 0.1;
|
||||
|
||||
MIGRATE_DIRECTORY="/mnt/migrator/DbScripts"
|
||||
LAST_MIGRATION_FILE="/mnt/data/last_migration"
|
||||
SERVER='mssql'
|
||||
DATABASE="vault_dev"
|
||||
USER="SA"
|
||||
PASSWD=$MSSQL_PASSWORD
|
||||
|
||||
while getopts "rs" arg; do
|
||||
while getopts "sp" arg; do
|
||||
case $arg in
|
||||
r)
|
||||
echo "Rerunning the last migration"
|
||||
RERUN=1
|
||||
;;
|
||||
s)
|
||||
echo "Running for self-host environment"
|
||||
LAST_MIGRATION_FILE="/mnt/data/last_self_host_migration"
|
||||
DATABASE="vault_dev_self_host"
|
||||
;;
|
||||
p)
|
||||
echo "Running for pipeline"
|
||||
MIGRATE_DIRECTORY=$MSSQL_MIGRATIONS_DIRECTORY
|
||||
SERVER=$MSSQL_HOST
|
||||
DATABASE=$MSSQL_DATABASE
|
||||
USER=$MSSQL_USER
|
||||
PASSWD=$MSSQL_PASS
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -f "$LAST_MIGRATION_FILE" ]; then
|
||||
echo "$LAST_MIGRATION_FILE not found!"
|
||||
echo "This will run all migrations which might cause unexpected behaviour if the database is not empty."
|
||||
echo
|
||||
read -p "Run all Migrations? (y/N) " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]
|
||||
then
|
||||
exit 1
|
||||
fi
|
||||
LAST_MIGRATION=""
|
||||
else
|
||||
LAST_MIGRATION=$(cat $LAST_MIGRATION_FILE)
|
||||
fi
|
||||
|
||||
[ -z "$LAST_MIGRATION" ]
|
||||
PERFORM_MIGRATION=$?
|
||||
|
||||
|
||||
# Create database if it does not already exist
|
||||
# Create databases if they do not already exist
|
||||
QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = '$DATABASE')
|
||||
BEGIN
|
||||
CREATE DATABASE $DATABASE;
|
||||
END;
|
||||
|
||||
IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE')
|
||||
BEGIN
|
||||
CREATE DATABASE migrations_$DATABASE;
|
||||
CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] (
|
||||
[Id] INT IDENTITY(1,1) PRIMARY KEY,
|
||||
[Filename] NVARCHAR(MAX) NOT NULL,
|
||||
[CreationDate] DATETIME2 (7) NULL,
|
||||
);
|
||||
END;"
|
||||
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
|
||||
|
||||
should_migrate () {
|
||||
local file=$(basename $1)
|
||||
local query="SELECT * FROM [migrations] WHERE [Filename] = '$file'"
|
||||
local result=$(/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query")
|
||||
if [[ "$result" =~ .*"$file".* ]]; then
|
||||
return 1;
|
||||
else
|
||||
return 0;
|
||||
fi
|
||||
}
|
||||
|
||||
record_migration () {
|
||||
echo "recording $1"
|
||||
local file=$(basename $1)
|
||||
echo $file
|
||||
local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())"
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query"
|
||||
}
|
||||
|
||||
migrate () {
|
||||
local file=$1
|
||||
echo "Performing $file"
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d $DATABASE -U $USER -P $PASSWD -I -i $file
|
||||
echo $file > $LAST_MIGRATION_FILE
|
||||
}
|
||||
|
||||
for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do
|
||||
if (( PERFORM_MIGRATION == 0 )); then
|
||||
BASENAME=$(basename $f)
|
||||
if should_migrate $f == 1 ; then
|
||||
migrate $f
|
||||
record_migration $f
|
||||
else
|
||||
echo "Skipping $f"
|
||||
if [ "$LAST_MIGRATION" == "$f" ]; then
|
||||
PERFORM_MIGRATION=0
|
||||
|
||||
# Rerun last migration
|
||||
if [ -n "$RERUN" ]; then
|
||||
migrate $f
|
||||
unset RERUN
|
||||
fi
|
||||
fi
|
||||
echo "Skipping $f, $BASENAME"
|
||||
fi
|
||||
done;
|
||||
|
||||
23
dev/migrate_migration_record.ps1
Executable file
23
dev/migrate_migration_record.ps1
Executable file
@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env pwsh
|
||||
# This script need only be run once
|
||||
#
|
||||
# This is a migration script for updating recording the last migration run
|
||||
# in a file to recording migrations in a database table. It will create a
|
||||
# migrations_vault table and store all of the previously run migrations as
|
||||
# indicated by a last_migrations file. It will then delete this file.
|
||||
|
||||
# Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use
|
||||
# the mssql-tools container which runs under x86_64. We should monitor this
|
||||
# in the future and investigate if we can migrate back.
|
||||
# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args
|
||||
|
||||
docker run `
|
||||
-v "$(pwd)/helpers/mssql:/mnt/helpers" `
|
||||
-v "$(pwd)/../util/Migrator:/mnt/migrator/" `
|
||||
-v "$(pwd)/.data/mssql:/mnt/data" `
|
||||
--env-file .env `
|
||||
--network=bitwardenserver_default `
|
||||
--rm `
|
||||
-it `
|
||||
mcr.microsoft.com/mssql-tools `
|
||||
/mnt/helpers/migrate_migrations.sh @args
|
||||
Loading…
Reference in New Issue
Block a user