1
0
mirror of https://github.com/bitwarden/server.git synced 2024-11-21 12:05:42 +01:00

[PM-2444] Add Pipeline for Testing All Database Variants in CI (#2471)

* Add Pipeline

* Fix Lint

* Added a Change

* Update Pipeline

* Add Multi-Version Support

* Use Profile Switch for each profile

* Fix MySql

* Debug MySql

* Use Proper Seperator

* Add Allow User Variables=true

* Pipeline Work

* Expand Config for Postgres

* Change Config Key

* Add Debug Step

* Fix Debug Step

* Fix Tests

* Add Sleep

* Fix Tests

* Fix SQL Server Tests

* Add Sqlite

* Use Context Property

* Fix Tests

* Fix Test Logger

* Update AccountRevisionDate Check

* Fix Postgres Time Issues

* Formatting and Pipeline Update

* Remove Unneeded SqlServer Setting

* Update .github/workflows/infrastructure-tests.yml

Co-authored-by: mimartin12 <77340197+mimartin12@users.noreply.github.com>

---------

Co-authored-by: mimartin12 <77340197+mimartin12@users.noreply.github.com>
This commit is contained in:
Justin Baur 2023-05-30 13:25:55 -04:00 committed by GitHub
parent 6e6432c1d0
commit 61a0efbdfc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 268 additions and 84 deletions

View File

@ -0,0 +1,117 @@
---
name: Run Database Infrastructure Tests
on:
pull_request:
branches-ignore:
- 'l10n_master'
- 'gh-pages'
paths:
- '.github/workflows/infrastructure-tests.yml' # This file
- 'src/Sql/**' # SQL Server Database Changes
- 'util/Migrator/**' # New SQL Server Migrations
- 'util/MySqlMigrations/**' # Changes to MySQL
- 'util/PostgresMigrations/**' # Changes to Postgres
- 'util/SqliteMigrations/**' # Changes to Sqlite
- 'src/Infrastructure.Dapper/**' # Changes to SQL Server Dapper Repository Layer
- 'src/Infrastructure.EntityFramework/**' # Changes to Entity Framework Repository Layer
- 'test/Infrastructure.IntegrationTest/**' # Any changes to the tests
push:
branches:
- 'master'
- 'rc'
paths:
- '.github/workflows/infrastructure-tests.yml' # This file
- 'src/Sql/**' # SQL Server Database Changes
- 'util/Migrator/**' # New SQL Server Migrations
- 'util/MySqlMigrations/**' # Changes to MySQL
- 'util/PostgresMigrations/**' # Changes to Postgres
- 'util/SqliteMigrations/**' # Changes to Sqlite
- 'src/Infrastructure.Dapper/**' # Changes to SQL Server Dapper Repository Layer
- 'src/Infrastructure.EntityFramework/**' # Changes to Entity Framework Repository Layer
- 'test/Infrastructure.IntegrationTest/**' # Any changes to the tests
workflow_dispatch:
inputs: {}
jobs:
test:
name: 'Run Infrastructure.IntegrationTest'
runs-on: ubuntu-22.04
steps:
- name: Checkout repo
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
- name: Set up dotnet
uses: actions/setup-dotnet@607fce577a46308457984d59e4954e075820f10a # v3.0.3
with:
dotnet-version: '6.0.x'
- name: Restore Tools
run: dotnet tool restore
- name: Compose Databases
working-directory: 'dev'
# We could think about not using profiles and pulling images directly to cover multiple versions
run: |
cp .env.example .env
docker compose --profile mssql --profile postgres --profile mysql up -d
shell: pwsh
# I've seen the SQL Server container not be ready for commands right after starting up and just needing a bit longer to be ready
- name: Sleep
run: sleep 15s
- name: Migrate SQL Server
working-directory: 'dev'
run: "pwsh ./migrate.ps1"
shell: pwsh
- name: Migrate MySQL
working-directory: 'util/MySqlMigrations'
run: 'dotnet ef database update --connection "$CONN_STR" -- --GlobalSettings:MySql:ConnectionString="$CONN_STR"'
env:
CONN_STR: "server=localhost;uid=root;pwd=SET_A_PASSWORD_HERE_123;database=vault_dev;Allow User Variables=true"
- name: Migrate Postgres
working-directory: 'util/PostgresMigrations'
run: 'dotnet ef database update --connection "$CONN_STR" -- --GlobalSettings:PostgreSql:ConnectionString="$CONN_STR"'
env:
CONN_STR: "Host=localhost;Username=postgres;Password=SET_A_PASSWORD_HERE_123;Database=vault_dev"
- name: Migrate Sqlite
working-directory: 'util/SqliteMigrations'
run: 'dotnet ef database update --connection "$CONN_STR" -- --GlobalSettings:Sqlite:ConnectionString="$CONN_STR"'
env:
CONN_STR: "Data Source=${{ runner.temp }}/test.db"
- name: Run Tests
working-directory: 'test/Infrastructure.IntegrationTest'
env:
# Default Postgres:
BW_TEST_DATABASES__0__TYPE: "Postgres"
BW_TEST_DATABASES__0__CONNECTIONSTRING: "Host=localhost;Username=postgres;Password=SET_A_PASSWORD_HERE_123;Database=vault_dev"
# Default MySql
BW_TEST_DATABASES__1__TYPE: "MySql"
BW_TEST_DATABASES__1__CONNECTIONSTRING: "server=localhost;uid=root;pwd=SET_A_PASSWORD_HERE_123;database=vault_dev"
# Default Dapper SqlServer
BW_TEST_DATABASES__2__TYPE: "SqlServer"
BW_TEST_DATABASES__2__CONNECTIONSTRING: "Server=localhost;Database=vault_dev;User Id=SA;Password=SET_A_PASSWORD_HERE_123;Encrypt=True;TrustServerCertificate=True;"
# Default Sqlite
BW_TEST_DATABASES__3__TYPE: "Sqlite"
BW_TEST_DATABASES__3__CONNECTIONSTRING: "Data Source=${{ runner.temp }}/test.db"
run: dotnet test --logger "trx;LogFileName=infrastructure-test-results.trx"
shell: pwsh
- name: Report test results
uses: dorny/test-reporter@c9b3d0e2bd2a4e96aaf424dbaa31c46b42318226 # v1.6.0
if: always()
with:
name: Test Results
path: "**/*-test-results.trx"
reporter: dotnet-trx
fail-on-error: true
- name: Docker compose down
if: always()
working-directory: "dev"
run: docker compose down
shell: pwsh

View File

@ -428,6 +428,8 @@ public class CollectionRepository : Repository<Core.Entities.Collection, Collect
// Remove all existing ones that are no longer requested
var requestedUserIds = requestedUsers.Select(u => u.Id);
dbContext.CollectionUsers.RemoveRange(existingCollectionUsers.Where(cu => !requestedUserIds.Contains(cu.OrganizationUserId)));
// Need to save the new collection users before running the bump revision code
await dbContext.SaveChangesAsync();
await dbContext.UserBumpAccountRevisionDateByCollectionIdAsync(id, organizationId);
await dbContext.SaveChangesAsync();
}

View File

@ -64,9 +64,10 @@ public static class DatabaseContextExtensions
from cg in cg_g.DefaultIfEmpty()
where ou.OrganizationId == organizationId &&
ou.Status == OrganizationUserStatusType.Confirmed &&
cg.CollectionId != null &&
ou.AccessAll == true &&
g.AccessAll == true
(cu.CollectionId != null ||
cg.CollectionId != null ||
ou.AccessAll == true ||
g.AccessAll == true)
select u;
var users = await query.ToListAsync();

View File

@ -112,6 +112,8 @@ public class CipherRepository : Repository<Core.Vault.Entities.Cipher, Cipher, G
var entity = Mapper.Map<Cipher>((Core.Vault.Entities.Cipher)cipher);
await dbContext.AddAsync(entity);
await dbContext.SaveChangesAsync();
if (cipher.OrganizationId.HasValue)
{
await dbContext.UserBumpAccountRevisionDateByCipherIdAsync(cipher.Id, cipher.OrganizationId.Value);

View File

@ -16,7 +16,7 @@ public class EmergencyAccessRepositoriesTests
var grantorUser = await userRepository.CreateAsync(new User
{
Name = "Test Grantor User",
Email = "test+grantor@email.com",
Email = $"test+grantor{Guid.NewGuid()}@email.com",
ApiKey = "TEST",
SecurityStamp = "stamp",
});
@ -24,7 +24,7 @@ public class EmergencyAccessRepositoriesTests
var granteeUser = await userRepository.CreateAsync(new User
{
Name = "Test Grantee User",
Email = "test+grantee@email.com",
Email = $"test+grantee{Guid.NewGuid()}@email.com",
ApiKey = "TEST",
SecurityStamp = "stamp",
});

View File

@ -1,17 +1,31 @@
using Microsoft.Extensions.Configuration;
using Bit.Core.Enums;
using Microsoft.Extensions.Configuration;
namespace Bit.Infrastructure.IntegrationTest;
public static class ConfigurationExtensions
public class Database
{
public static bool TryGetConnectionString(this IConfiguration config, string key, out string connectionString)
{
connectionString = config[key];
if (string.IsNullOrEmpty(connectionString))
{
return false;
public SupportedDatabaseProviders Type { get; set; }
public string ConnectionString { get; set; } = default!;
public bool UseEf { get; set; }
public bool Enabled { get; set; } = true;
}
return true;
internal class TypedConfig
{
public Database[] Databases { get; set; } = default!;
}
public static class ConfigurationExtensions
{
public static Database[] GetDatabases(this IConfiguration config)
{
var typedConfig = config.Get<TypedConfig>();
if (typedConfig.Databases == null)
{
return Array.Empty<Database>();
}
return typedConfig.Databases.Where(d => d.Enabled).ToArray();
}
}

View File

@ -3,6 +3,7 @@ using Bit.Core.Enums;
using Bit.Core.Settings;
using Bit.Infrastructure.Dapper;
using Bit.Infrastructure.EntityFramework;
using Bit.Infrastructure.EntityFramework.Repositories;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
@ -45,7 +46,11 @@ public class DatabaseDataAttribute : DataAttribute
}
};
if (config.TryGetConnectionString(DatabaseTheoryAttribute.DapperSqlServerKey, out var dapperSqlServerConnectionString))
var databases = config.GetDatabases();
foreach (var database in databases)
{
if (database.Type == SupportedDatabaseProviders.SqlServer && !database.UseEf)
{
var dapperSqlServerCollection = new ServiceCollection();
dapperSqlServerCollection.AddLogging(configureLogging);
@ -55,32 +60,25 @@ public class DatabaseDataAttribute : DataAttribute
DatabaseProvider = "sqlServer",
SqlServer = new GlobalSettings.SqlSettings
{
ConnectionString = dapperSqlServerConnectionString,
ConnectionString = database.ConnectionString,
},
};
dapperSqlServerCollection.AddSingleton(globalSettings);
dapperSqlServerCollection.AddSingleton<IGlobalSettings>(globalSettings);
dapperSqlServerCollection.AddSingleton<ITestDatabaseHelper>(_ => new DapperSqlServerTestDatabaseHelper(database));
dapperSqlServerCollection.AddDataProtection();
yield return dapperSqlServerCollection.BuildServiceProvider();
}
if (config.TryGetConnectionString(DatabaseTheoryAttribute.EfPostgresKey, out var efPostgresConnectionString))
else
{
var efPostgresCollection = new ServiceCollection();
efPostgresCollection.AddLogging(configureLogging);
efPostgresCollection.SetupEntityFramework(efPostgresConnectionString, SupportedDatabaseProviders.Postgres);
efPostgresCollection.AddPasswordManagerEFRepositories(SelfHosted);
efPostgresCollection.AddTransient<ITestDatabaseHelper, EfTestDatabaseHelper>();
yield return efPostgresCollection.BuildServiceProvider();
}
if (config.TryGetConnectionString(DatabaseTheoryAttribute.EfMySqlKey, out var efMySqlConnectionString))
{
var efMySqlCollection = new ServiceCollection();
efMySqlCollection.AddLogging(configureLogging);
efMySqlCollection.SetupEntityFramework(efMySqlConnectionString, SupportedDatabaseProviders.MySql);
efMySqlCollection.AddPasswordManagerEFRepositories(SelfHosted);
efMySqlCollection.AddTransient<ITestDatabaseHelper, EfTestDatabaseHelper>();
yield return efMySqlCollection.BuildServiceProvider();
var efCollection = new ServiceCollection();
efCollection.AddLogging(configureLogging);
efCollection.SetupEntityFramework(database.ConnectionString, database.Type);
efCollection.AddPasswordManagerEFRepositories(SelfHosted);
efCollection.AddTransient<ITestDatabaseHelper>(sp => new EfTestDatabaseHelper(sp.GetRequiredService<DatabaseContext>(), database));
efCollection.AddDataProtection();
yield return efCollection.BuildServiceProvider();
}
}
}
}

View File

@ -7,24 +7,18 @@ public class DatabaseTheoryAttribute : TheoryAttribute
{
private static IConfiguration? _cachedConfiguration;
public const string DapperSqlServerKey = "Dapper:SqlServer";
public const string EfPostgresKey = "Ef:Postgres";
public const string EfMySqlKey = "Ef:MySql";
public DatabaseTheoryAttribute()
{
if (!HasAnyDatabaseSetup())
{
Skip = "No database connections strings setup.";
Skip = "No databases setup.";
}
}
private static bool HasAnyDatabaseSetup()
{
var config = GetConfiguration();
return config.TryGetConnectionString(DapperSqlServerKey, out _) ||
config.TryGetConnectionString(EfPostgresKey, out _) ||
config.TryGetConnectionString(EfMySqlKey, out _);
return config.GetDatabases().Length > 0;
}
public static IConfiguration GetConfiguration()

View File

@ -16,7 +16,7 @@ public class OrganizationUserRepositoryTests
var user = await userRepository.CreateAsync(new User
{
Name = "Test User",
Email = "test@email.com",
Email = $"test+{Guid.NewGuid()}@email.com",
ApiKey = "TEST",
SecurityStamp = "stamp",
});
@ -24,6 +24,8 @@ public class OrganizationUserRepositoryTests
var organization = await organizationRepository.CreateAsync(new Organization
{
Name = "Test Org",
BillingEmail = user.Email, // TODO: EF does not enfore this being NOT NULL
Plan = "Test", // TODO: EF does not enforce this being NOT NULl
});
var orgUser = await organizationUserRepository.CreateAsync(new OrganizationUser
@ -50,7 +52,7 @@ public class OrganizationUserRepositoryTests
var user1 = await userRepository.CreateAsync(new User
{
Name = "Test User 1",
Email = "test1@email.com",
Email = $"test+{Guid.NewGuid()}@email.com",
ApiKey = "TEST",
SecurityStamp = "stamp",
});
@ -58,7 +60,7 @@ public class OrganizationUserRepositoryTests
var user2 = await userRepository.CreateAsync(new User
{
Name = "Test User 2",
Email = "test1@email.com",
Email = $"test+{Guid.NewGuid()}@email.com",
ApiKey = "TEST",
SecurityStamp = "stamp",
});
@ -66,18 +68,22 @@ public class OrganizationUserRepositoryTests
var organization = await organizationRepository.CreateAsync(new Organization
{
Name = "Test Org",
BillingEmail = user1.Email, // TODO: EF does not enforce this being NOT NULl
Plan = "Test", // TODO: EF does not enforce this being NOT NULl
});
var orgUser1 = await organizationUserRepository.CreateAsync(new OrganizationUser
{
OrganizationId = organization.Id,
UserId = user1.Id,
Status = OrganizationUserStatusType.Confirmed,
});
var orgUser2 = await organizationUserRepository.CreateAsync(new OrganizationUser
{
OrganizationId = organization.Id,
UserId = user2.Id,
Status = OrganizationUserStatusType.Confirmed,
});
helper.ClearTracker();

View File

@ -4,6 +4,7 @@ namespace Bit.Infrastructure.IntegrationTest;
public interface ITestDatabaseHelper
{
Database Info { get; }
void ClearTracker();
}
@ -11,11 +12,14 @@ public class EfTestDatabaseHelper : ITestDatabaseHelper
{
private readonly DatabaseContext _databaseContext;
public EfTestDatabaseHelper(DatabaseContext databaseContext)
public EfTestDatabaseHelper(DatabaseContext databaseContext, Database database)
{
_databaseContext = databaseContext;
Info = database;
}
public Database Info { get; }
public void ClearTracker()
{
_databaseContext.ChangeTracker.Clear();
@ -24,11 +28,13 @@ public class EfTestDatabaseHelper : ITestDatabaseHelper
public class DapperSqlServerTestDatabaseHelper : ITestDatabaseHelper
{
public DapperSqlServerTestDatabaseHelper()
public DapperSqlServerTestDatabaseHelper(Database database)
{
Info = database;
}
public Database Info { get; }
public void ClearTracker()
{
// There are no tracked entities in Dapper SQL Server

View File

@ -1,5 +1,6 @@
using Bit.Core.Entities;
using Bit.Core.Enums;
using Bit.Core.Models.Data;
using Bit.Core.Repositories;
using Bit.Core.Vault.Entities;
using Bit.Core.Vault.Enums;
@ -20,7 +21,7 @@ public class CipherRepositoryTests
var user = await userRepository.CreateAsync(new User
{
Name = "Test User",
Email = "test@email.com",
Email = $"test+{Guid.NewGuid()}@email.com",
ApiKey = "TEST",
SecurityStamp = "stamp",
});
@ -29,6 +30,7 @@ public class CipherRepositoryTests
{
Type = CipherType.Login,
UserId = user.Id,
Data = "", // TODO: EF does not enforce this as NOT NULL
});
helper.ClearTracker();
@ -55,21 +57,27 @@ public class CipherRepositoryTests
var user = await userRepository.CreateAsync(new User
{
Name = "Test User",
Email = "test@email.com",
Email = $"test+{Guid.NewGuid()}@email.com",
ApiKey = "TEST",
SecurityStamp = "stamp",
});
helper.ClearTracker();
user = await userRepository.GetByIdAsync(user.Id);
var organization = await organizationRepository.CreateAsync(new Organization
{
Name = "Test Organization",
BillingEmail = user.Email,
Plan = "Test" // TODO: EF does not enforce this as NOT NULL
});
await organizationUserRepository.CreateAsync(new OrganizationUser
var orgUser = await organizationUserRepository.CreateAsync(new OrganizationUser
{
UserId = user.Id,
OrganizationId = organization.Id,
Status = OrganizationUserStatusType.Accepted,
Status = OrganizationUserStatusType.Confirmed,
Type = OrganizationUserType.Owner,
});
@ -79,12 +87,27 @@ public class CipherRepositoryTests
OrganizationId = organization.Id
});
await Task.Delay(100);
await collectionRepository.UpdateUsersAsync(collection.Id, new[]
{
new CollectionAccessSelection
{
Id = orgUser.Id,
HidePasswords = true,
ReadOnly = true,
},
});
helper.ClearTracker();
await Task.Delay(100);
await cipherRepository.CreateAsync(new CipherDetails
{
Type = CipherType.Login,
OrganizationId = organization.Id,
Data = "", // TODO: EF does not enforce this as NOT NULL
}, new List<Guid>
{
collection.Id,
@ -92,7 +115,8 @@ public class CipherRepositoryTests
var updatedUser = await userRepository.GetByIdAsync(user.Id);
Assert.NotEqual(updatedUser.AccountRevisionDate, user.AccountRevisionDate);
Assert.True(updatedUser.AccountRevisionDate - user.AccountRevisionDate > TimeSpan.Zero,
"The AccountRevisionDate is expected to be changed");
var collectionCiphers = await collectionCipherRepository.GetManyByOrganizationIdAsync(organization.Id);
Assert.NotEmpty(collectionCiphers);

View File

@ -7,15 +7,20 @@ using Microsoft.Extensions.DependencyInjection;
namespace Bit.MySqlMigrations;
public static class GlobalSettingsFactory
public class GlobalSettingsFactory
{
public static GlobalSettings GlobalSettings { get; } = new GlobalSettings();
static GlobalSettingsFactory()
public GlobalSettings GlobalSettings { get; }
public GlobalSettingsFactory(string[] args)
{
GlobalSettings = new GlobalSettings();
// UserSecretsId here should match what is in Api.csproj
var configBuilder = new ConfigurationBuilder().AddUserSecrets("bitwarden-Api");
var Configuration = configBuilder.Build();
ConfigurationBinder.Bind(Configuration.GetSection("GlobalSettings"), GlobalSettings);
var config = new ConfigurationBuilder()
.AddUserSecrets("bitwarden-Api")
.AddCommandLine(args)
.Build();
config.GetSection("GlobalSettings").Bind(GlobalSettings);
}
}
@ -27,7 +32,9 @@ public class DatabaseContextFactory : IDesignTimeDbContextFactory<DatabaseContex
services.AddDataProtection();
var serviceProvider = services.BuildServiceProvider();
var globalSettings = GlobalSettingsFactory.GlobalSettings;
var globalSettings = new GlobalSettingsFactory(args)
.GlobalSettings;
var optionsBuilder = new DbContextOptionsBuilder<DatabaseContext>();
var connectionString = globalSettings.MySql?.ConnectionString;
if (string.IsNullOrWhiteSpace(connectionString))

View File

@ -7,15 +7,20 @@ using Microsoft.Extensions.DependencyInjection;
namespace Bit.PostgresMigrations;
public static class GlobalSettingsFactory
public class GlobalSettingsFactory
{
public static GlobalSettings GlobalSettings { get; } = new GlobalSettings();
static GlobalSettingsFactory()
public GlobalSettings GlobalSettings { get; }
public GlobalSettingsFactory(string[] args)
{
GlobalSettings = new GlobalSettings();
// UserSecretsId here should match what is in Api.csproj
var configBuilder = new ConfigurationBuilder().AddUserSecrets("bitwarden-Api");
var Configuration = configBuilder.Build();
ConfigurationBinder.Bind(Configuration.GetSection("GlobalSettings"), GlobalSettings);
var config = new ConfigurationBuilder()
.AddUserSecrets("bitwarden-Api")
.AddCommandLine(args)
.Build();
config.GetSection("GlobalSettings").Bind(GlobalSettings);
}
}
@ -27,7 +32,8 @@ public class DatabaseContextFactory : IDesignTimeDbContextFactory<DatabaseContex
services.AddDataProtection();
var serviceProvider = services.BuildServiceProvider();
var globalSettings = GlobalSettingsFactory.GlobalSettings;
var globalSettings = new GlobalSettingsFactory(args)
.GlobalSettings;
var optionsBuilder = new DbContextOptionsBuilder<DatabaseContext>();
var connectionString = globalSettings.PostgreSql?.ConnectionString;
if (string.IsNullOrWhiteSpace(connectionString))

View File

@ -7,14 +7,20 @@ using Microsoft.Extensions.DependencyInjection;
namespace Bit.SqliteMigrations;
public static class GlobalSettingsFactory
public class GlobalSettingsFactory
{
public static GlobalSettings GlobalSettings { get; } = new GlobalSettings();
static GlobalSettingsFactory()
public GlobalSettings GlobalSettings { get; }
public GlobalSettingsFactory(string[] args)
{
var configBuilder = new ConfigurationBuilder().AddUserSecrets("bitwarden-Api");
var Configuration = configBuilder.Build();
ConfigurationBinder.Bind(Configuration.GetSection("GlobalSettings"), GlobalSettings);
GlobalSettings = new GlobalSettings();
// UserSecretsId here should match what is in Api.csproj
var config = new ConfigurationBuilder()
.AddUserSecrets("bitwarden-Api")
.AddCommandLine(args)
.Build();
config.GetSection("GlobalSettings").Bind(GlobalSettings);
}
}
@ -26,7 +32,8 @@ public class DatabaseContextFactory : IDesignTimeDbContextFactory<DatabaseContex
services.AddDataProtection();
var serviceProvider = services.BuildServiceProvider();
var globalSettings = GlobalSettingsFactory.GlobalSettings;
var globalSettings = new GlobalSettingsFactory(args)
.GlobalSettings;
var optionsBuilder = new DbContextOptionsBuilder<DatabaseContext>();
var connectionString = globalSettings.Sqlite?.ConnectionString ?? "Data Source=:memory:";
if (string.IsNullOrWhiteSpace(connectionString))