mirror of
https://github.com/bitwarden/server.git
synced 2024-11-22 12:15:36 +01:00
[BEEEP] Use MsSqlMigratorUtility for local development databases (#3850)
* Update migrate.ps1 to use MsSqlMigratorUtility for dev databases * Remove old handwritten scripts * Migrate existing migration records * Update Github Workflow to call MsSqlMigratorUtility directly
This commit is contained in:
Thomas Rittson
GitHub
parent
dd21d8fcf4
commit
386ff744ef
12
.github/workflows/test-database.yml
vendored
12
.github/workflows/test-database.yml
vendored
@ -57,9 +57,9 @@ jobs:
|
||||
run: sleep 15s
|
||||
|
||||
- name: Migrate SQL Server
|
||||
working-directory: "dev"
|
||||
run: "./migrate.ps1"
|
||||
shell: pwsh
|
||||
run: 'dotnet run --project util/MsSqlMigratorUtility/ "$CONN_STR"'
|
||||
env:
|
||||
CONN_STR: "Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;"
|
||||
|
||||
- name: Migrate MySQL
|
||||
working-directory: "util/MySqlMigrations"
|
||||
@ -147,9 +147,9 @@ jobs:
|
||||
shell: pwsh
|
||||
|
||||
- name: Migrate
|
||||
working-directory: "dev"
|
||||
run: "./migrate.ps1"
|
||||
shell: pwsh
|
||||
run: 'dotnet run --project util/MsSqlMigratorUtility/ "$CONN_STR"'
|
||||
env:
|
||||
CONN_STR: "Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;"
|
||||
|
||||
- name: Diff .sqlproj to migrations
|
||||
run: /usr/local/sqlpackage/sqlpackage /action:DeployReport /SourceFile:"Sql.dacpac" /TargetConnectionString:"Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;" /OutputPath:"report.xml" /p:IgnoreColumnOrder=True /p:IgnoreComments=True
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
#
|
||||
# !!! UPDATED 2024 for MsSqlMigratorUtility !!!
|
||||
#
|
||||
# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294)
|
||||
# where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost.
|
||||
# The best workaround seems to be adding tiny delay like so:
|
||||
sleep 0.1;
|
||||
|
||||
MIGRATE_DIRECTORY="/mnt/migrator/DbScripts"
|
||||
LAST_MIGRATION_FILE="/mnt/data/last_migration"
|
||||
SERVER='mssql'
|
||||
DATABASE="vault_dev"
|
||||
USER="SA"
|
||||
@ -16,58 +16,33 @@ while getopts "s" arg; do
|
||||
case $arg in
|
||||
s)
|
||||
echo "Running for self-host environment"
|
||||
LAST_MIGRATION_FILE="/mnt/data/last_self_host_migration"
|
||||
DATABASE="vault_dev_self_host"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -f "$LAST_MIGRATION_FILE" ]; then
|
||||
echo "No migration file, nothing to migrate to a database store"
|
||||
exit 1
|
||||
else
|
||||
LAST_MIGRATION=$(cat $LAST_MIGRATION_FILE)
|
||||
rm $LAST_MIGRATION_FILE
|
||||
fi
|
||||
|
||||
[ -z "$LAST_MIGRATION" ]
|
||||
PERFORM_MIGRATION=$?
|
||||
|
||||
# Create database if it does not already exist
|
||||
QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE')
|
||||
QUERY="IF OBJECT_ID('[$DATABASE].[dbo].[Migration]') IS NULL AND OBJECT_ID('[migrations_$DATABASE].[dbo].[migrations]') IS NOT NULL
|
||||
BEGIN
|
||||
CREATE DATABASE migrations_$DATABASE;
|
||||
END;
|
||||
-- Create [database].dbo.Migration with the schema expected by MsSqlMigratorUtility
|
||||
SET ANSI_NULLS ON;
|
||||
SET QUOTED_IDENTIFIER ON;
|
||||
|
||||
CREATE TABLE [$DATABASE].[dbo].[Migration](
|
||||
[Id] [int] IDENTITY(1,1) NOT NULL,
|
||||
[ScriptName] [nvarchar](255) NOT NULL,
|
||||
[Applied] [datetime] NOT NULL
|
||||
) ON [PRIMARY];
|
||||
|
||||
ALTER TABLE [$DATABASE].[dbo].[Migration] ADD CONSTRAINT [PK_Migration_Id] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[Id] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY];
|
||||
|
||||
-- Copy across old data
|
||||
INSERT INTO [$DATABASE].[dbo].[Migration] (ScriptName, Applied)
|
||||
SELECT CONCAT('Bit.Migrator.DbScripts.', [Filename]), CreationDate
|
||||
FROM [migrations_$DATABASE].[dbo].[migrations];
|
||||
END
|
||||
"
|
||||
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
|
||||
|
||||
QUERY="IF OBJECT_ID('[dbo].[migrations_$DATABASE]') IS NULL
|
||||
BEGIN
|
||||
CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] (
|
||||
[Id] INT IDENTITY(1,1) PRIMARY KEY,
|
||||
[Filename] NVARCHAR(MAX) NOT NULL,
|
||||
[CreationDate] DATETIME2 (7) NULL,
|
||||
);
|
||||
END;"
|
||||
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
|
||||
|
||||
record_migration () {
|
||||
echo "recording $1"
|
||||
local file=$(basename $1)
|
||||
echo $file
|
||||
local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())"
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query"
|
||||
}
|
||||
|
||||
for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do
|
||||
if (( PERFORM_MIGRATION == 0 )); then
|
||||
echo "Still need to migrate $f"
|
||||
else
|
||||
record_migration $f
|
||||
if [ "$LAST_MIGRATION" == "$f" ]; then
|
||||
PERFORM_MIGRATION=0
|
||||
fi
|
||||
fi
|
||||
done;
|
||||
|
||||
@ -1,94 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# There seems to be [a bug with docker-compose](https://github.com/docker/compose/issues/4076#issuecomment-324932294)
|
||||
# where it takes ~40ms to connect to the terminal output of the container, so stuff logged to the terminal in this time is lost.
|
||||
# The best workaround seems to be adding tiny delay like so:
|
||||
sleep 0.1;
|
||||
|
||||
MIGRATE_DIRECTORY="/mnt/migrator/DbScripts"
|
||||
SERVER='mssql'
|
||||
DATABASE="vault_dev"
|
||||
USER="SA"
|
||||
PASSWD=$MSSQL_PASSWORD
|
||||
|
||||
while getopts "sp" arg; do
|
||||
case $arg in
|
||||
s)
|
||||
echo "Running for self-host environment"
|
||||
DATABASE="vault_dev_self_host"
|
||||
;;
|
||||
p)
|
||||
echo "Running for pipeline"
|
||||
MIGRATE_DIRECTORY=$MSSQL_MIGRATIONS_DIRECTORY
|
||||
SERVER=$MSSQL_HOST
|
||||
DATABASE=$MSSQL_DATABASE
|
||||
USER=$MSSQL_USER
|
||||
PASSWD=$MSSQL_PASS
|
||||
esac
|
||||
done
|
||||
|
||||
# Create databases if they do not already exist
|
||||
QUERY="IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = '$DATABASE')
|
||||
BEGIN
|
||||
CREATE DATABASE $DATABASE;
|
||||
END;
|
||||
|
||||
GO
|
||||
IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'migrations_$DATABASE')
|
||||
BEGIN
|
||||
CREATE DATABASE migrations_$DATABASE;
|
||||
END;
|
||||
|
||||
GO
|
||||
"
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d master -U $USER -P $PASSWD -I -Q "$QUERY"
|
||||
echo "Return code: $?"
|
||||
|
||||
# Create migrations table if it does not already exist
|
||||
QUERY="IF OBJECT_ID('[migrations_$DATABASE].[dbo].[migrations]') IS NULL
|
||||
BEGIN
|
||||
CREATE TABLE [migrations_$DATABASE].[dbo].[migrations] (
|
||||
[Id] INT IDENTITY(1,1) PRIMARY KEY,
|
||||
[Filename] NVARCHAR(MAX) NOT NULL,
|
||||
[CreationDate] DATETIME2 (7) NULL,
|
||||
);
|
||||
END;
|
||||
GO
|
||||
"
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$QUERY"
|
||||
echo "Return code: $?"
|
||||
|
||||
should_migrate () {
|
||||
local file=$(basename $1)
|
||||
local query="SELECT * FROM [migrations] WHERE [Filename] = '$file'"
|
||||
local result=$(/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query")
|
||||
if [[ "$result" =~ .*"$file".* ]]; then
|
||||
return 1;
|
||||
else
|
||||
return 0;
|
||||
fi
|
||||
}
|
||||
|
||||
record_migration () {
|
||||
echo "recording $1"
|
||||
local file=$(basename $1)
|
||||
echo $file
|
||||
local query="INSERT INTO [migrations] ([Filename], [CreationDate]) VALUES ('$file', GETUTCDATE())"
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d migrations_$DATABASE -U $USER -P $PASSWD -I -Q "$query"
|
||||
}
|
||||
|
||||
migrate () {
|
||||
local file=$1
|
||||
echo "Performing $file"
|
||||
/opt/mssql-tools/bin/sqlcmd -S $SERVER -d $DATABASE -U $USER -P $PASSWD -I -i $file
|
||||
}
|
||||
|
||||
for f in `ls -v $MIGRATE_DIRECTORY/*.sql`; do
|
||||
BASENAME=$(basename $f)
|
||||
if should_migrate $f == 1 ; then
|
||||
migrate $f
|
||||
record_migration $f
|
||||
else
|
||||
echo "Skipping $f, $BASENAME"
|
||||
fi
|
||||
done;
|
||||
@ -2,20 +2,20 @@
|
||||
# Creates the vault_dev database, and runs all the migrations.
|
||||
|
||||
# Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use
|
||||
# the mssql-tools container which runs under x86_64. We should monitor this
|
||||
# in the future and investigate if we can migrate back.
|
||||
# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args
|
||||
# the mssql-tools container which runs under x86_64.
|
||||
|
||||
param(
|
||||
[switch]$all = $false,
|
||||
[switch]$postgres = $false,
|
||||
[switch]$mysql = $false,
|
||||
[switch]$mssql = $false,
|
||||
[switch]$sqlite = $false,
|
||||
[switch]$selfhost = $false,
|
||||
[switch]$pipeline = $false
|
||||
[switch]$all,
|
||||
[switch]$postgres,
|
||||
[switch]$mysql,
|
||||
[switch]$mssql,
|
||||
[switch]$sqlite,
|
||||
[switch]$selfhost
|
||||
)
|
||||
|
||||
# Abort on any error
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
if (!$all -and !$postgres -and !$mysql -and !$sqlite) {
|
||||
$mssql = $true;
|
||||
}
|
||||
@ -29,22 +29,27 @@ if ($all -or $postgres -or $mysql -or $sqlite) {
|
||||
}
|
||||
|
||||
if ($all -or $mssql) {
|
||||
if ($selfhost) {
|
||||
$migrationArgs = "-s"
|
||||
} elseif ($pipeline) {
|
||||
$migrationArgs = "-p"
|
||||
function Get-UserSecrets {
|
||||
return dotnet user-secrets list --json --project ../src/Api | ConvertFrom-Json
|
||||
}
|
||||
|
||||
Write-Host "Starting Microsoft SQL Server Migrations"
|
||||
docker run `
|
||||
-v "$(pwd)/helpers/mssql:/mnt/helpers" `
|
||||
-v "$(pwd)/../util/Migrator:/mnt/migrator/" `
|
||||
-v "$(pwd)/.data/mssql:/mnt/data" `
|
||||
--env-file .env `
|
||||
--network=bitwardenserver_default `
|
||||
--rm `
|
||||
mcr.microsoft.com/mssql-tools `
|
||||
/mnt/helpers/run_migrations.sh $migrationArgs
|
||||
if ($selfhost) {
|
||||
$msSqlConnectionString = $(Get-UserSecrets).'dev:selfHostOverride:globalSettings:sqlServer:connectionString'
|
||||
$envName = "self-host"
|
||||
|
||||
Write-Output "Migrating your migrations to use MsSqlMigratorUtility (if needed)"
|
||||
./migrate_migration_record.ps1 -s
|
||||
} else {
|
||||
$msSqlConnectionString = $(Get-UserSecrets).'globalSettings:sqlServer:connectionString'
|
||||
$envName = "cloud"
|
||||
|
||||
Write-Output "Migrating your migrations to use MsSqlMigratorUtility (if needed)"
|
||||
./migrate_migration_record.ps1
|
||||
}
|
||||
|
||||
Write-Host "Starting Microsoft SQL Server Migrations for $envName"
|
||||
|
||||
dotnet run --project ../util/MsSqlMigratorUtility/ "$msSqlConnectionString"
|
||||
}
|
||||
|
||||
$currentDir = Get-Location
|
||||
|
||||
@ -1,15 +1,13 @@
|
||||
#!/usr/bin/env pwsh
|
||||
# This script need only be run once
|
||||
# !!! UPDATED 2024 for MsSqlMigratorUtility !!!
|
||||
#
|
||||
# This is a migration script for updating recording the last migration run
|
||||
# in a file to recording migrations in a database table. It will create a
|
||||
# migrations_vault table and store all of the previously run migrations as
|
||||
# indicated by a last_migrations file. It will then delete this file.
|
||||
# This is a migration script to move data from [migrations_vault_dev].[dbo].[migrations] (used by our custom
|
||||
# migrator script) to [vault_dev].[dbo].[Migration] (used by MsSqlMigratorUtility). It is safe to run multiple
|
||||
# times because it will not perform any migration if it detects that the new table is already present.
|
||||
# This will be deleted after a few months after everyone has (presumably) migrated to the new schema.
|
||||
|
||||
# Due to azure-edge-sql not containing the mssql-tools on ARM, we manually use
|
||||
# the mssql-tools container which runs under x86_64. We should monitor this
|
||||
# in the future and investigate if we can migrate back.
|
||||
# docker-compose --profile mssql exec mssql bash /mnt/helpers/run_migrations.sh @args
|
||||
# the mssql-tools container which runs under x86_64.
|
||||
|
||||
docker run `
|
||||
-v "$(pwd)/helpers/mssql:/mnt/helpers" `
|
||||
|
||||
Loading…
Reference in New Issue
Block a user