Upstream/bitwarden-server
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2025-01-22 21:51:22 +01:00
Code Issues Projects Releases Wiki Activity

Move jobs to api hosted service w/ quartz

Browse Source
This commit is contained in:
Kyle Spearrin 2018-08-09 16:08:09 -04:00
parent 7424f6a6dd
commit 10a19c90d3
21 changed files with 248 additions and 339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
bitwarden-core.sln
View File

@ -37,8 +37,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Setup", "util\Setup\Setup.c
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Server", "util\Server\Server.csproj", "{66B0A682-658A-4A82-B606-A077A4871448}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jobs", "src\Jobs\Jobs.csproj", "{7DCEBD8F-E5F3-4A3C-BD35-B64341590B74}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Function", "util\Function\Function.csproj", "{A6C44A84-8E51-4C64-B9C4-7B7C23253345}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Icons", "src\Icons\Icons.csproj", "{9CF59342-3912-4B45-A2BA-0F173666586D}"
@ -49,7 +47,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EventsProcessor", "src\Even
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Admin", "src\Admin\Admin.csproj", "{B131CEF3-89FB-4C90-ADB0-9E9C4246EB56}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Hub", "src\Hub\Hub.csproj", "{28635027-20E5-42FA-B218-B6C878DE5350}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Hub", "src\Hub\Hub.csproj", "{28635027-20E5-42FA-B218-B6C878DE5350}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -91,10 +89,6 @@ Global
{66B0A682-658A-4A82-B606-A077A4871448}.Debug|Any CPU.Build.0 = Debug|Any CPU
{66B0A682-658A-4A82-B606-A077A4871448}.Release|Any CPU.ActiveCfg = Release|Any CPU
{66B0A682-658A-4A82-B606-A077A4871448}.Release|Any CPU.Build.0 = Release|Any CPU
{7DCEBD8F-E5F3-4A3C-BD35-B64341590B74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{7DCEBD8F-E5F3-4A3C-BD35-B64341590B74}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7DCEBD8F-E5F3-4A3C-BD35-B64341590B74}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7DCEBD8F-E5F3-4A3C-BD35-B64341590B74}.Release|Any CPU.Build.0 = Release|Any CPU
{A6C44A84-8E51-4C64-B9C4-7B7C23253345}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A6C44A84-8E51-4C64-B9C4-7B7C23253345}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A6C44A84-8E51-4C64-B9C4-7B7C23253345}.Release|Any CPU.ActiveCfg = Release|Any CPU
@ -132,7 +126,6 @@ Global
{04148736-3C0B-445E-8B74-2020E7A53502} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84D}
{EF2164EF-1FC0-4518-A2ED-CE02D3630B00} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84E}
{66B0A682-658A-4A82-B606-A077A4871448} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84E}
{7DCEBD8F-E5F3-4A3C-BD35-B64341590B74} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84D}
{A6C44A84-8E51-4C64-B9C4-7B7C23253345} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84E}
{9CF59342-3912-4B45-A2BA-0F173666586D} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84D}
{994DD611-F266-4BD3-8072-3B1B57267ED5} = {DD5BD056-4AAE-43EF-BBD2-0B569B8DA84D}

1
src/Api/Api.csproj
View File

@ -18,6 +18,7 @@
<PackageReference Include="Microsoft.AspNetCore.App" Version="2.1.2" />
<PackageReference Include="IdentityServer4.AccessTokenValidation" Version="2.6.0" />
<PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv" Version="2.1.2" />
<PackageReference Include="Quartz" Version="3.0.6" />
</ItemGroup>
</Project>

5
src/Api/Dockerfile
View File

@ -4,7 +4,6 @@ LABEL com.bitwarden.product="bitwarden"
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
cron \
gosu \
&& rm -rf /var/lib/apt/lists/*
@ -12,11 +11,7 @@ ENV ASPNETCORE_URLS http://+:5000
WORKDIR /app
EXPOSE 5000
COPY obj/Docker/publish/Api .
COPY obj/Docker/publish/Jobs /jobs
COPY entrypoint.sh /
RUN chmod +x /entrypoint.sh
RUN mv /jobs/crontab /etc/cron.d/bitwarden-cron \
&& chmod 0644 /etc/cron.d/bitwarden-cron
ENTRYPOINT ["/entrypoint.sh"]

23
src/Api/Jobs/AliveJob.cs Normal file
View File

@ -0,0 +1,23 @@
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Quartz;
namespace Bit.Api.Jobs
{
public class AliveJob : IJob
{
private readonly ILogger<AliveJob> _logger;
public AliveJob(
ILogger<AliveJob> logger)
{
_logger = logger;
}
public Task Execute(IJobExecutionContext context)
{
_logger.LogInformation("It's alive!");
return Task.FromResult(0);
}
}
}

27
src/Api/Jobs/JobFactory.cs Normal file
View File

@ -0,0 +1,27 @@
using System;
using Quartz;
using Quartz.Spi;
namespace Bit.Api.Jobs
{
public class JobFactory : IJobFactory
{
private readonly IServiceProvider _container;
public JobFactory(IServiceProvider container)
{
_container = container;
}
public IJob NewJob(TriggerFiredBundle bundle, IScheduler scheduler)
{
return _container.GetService(bundle.JobDetail.JobType) as IJob;
}
public void ReturnJob(IJob job)
{
var disposable = job as IDisposable;
disposable?.Dispose();
}
}
}

40
src/Api/Jobs/JobListener.cs Normal file
View File

@ -0,0 +1,40 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Quartz;
namespace Bit.Api.Jobs
{
public class JobListener : IJobListener
{
private readonly ILogger<JobListener> _logger;
public JobListener(ILogger<JobListener> logger)
{
_logger = logger;
}
public string Name => "JobListener";
public Task JobExecutionVetoed(IJobExecutionContext context,
CancellationToken cancellationToken = default(CancellationToken))
{
return Task.FromResult(0);
}
public Task JobToBeExecuted(IJobExecutionContext context,
CancellationToken cancellationToken = default(CancellationToken))
{
_logger.LogInformation("Starting job {0} at {1}.", context.JobDetail.JobType.Name, DateTime.UtcNow);
return Task.FromResult(0);
}
public Task JobWasExecuted(IJobExecutionContext context, JobExecutionException jobException,
CancellationToken cancellationToken = default(CancellationToken))
{
_logger.LogInformation("Finished job {0} at {1}.", context.JobDetail.JobType.Name, DateTime.UtcNow);
return Task.FromResult(0);
}
}
}

80
src/Api/Jobs/JobsHostedService.cs Normal file
View File

@ -0,0 +1,80 @@
using System;
using System.Collections.Specialized;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Quartz;
using Quartz.Impl;
using Quartz.Impl.Matchers;
namespace Bit.Api.Jobs
{
public class JobsHostedService : IHostedService, IDisposable
{
private readonly IServiceProvider _serviceProvider;
private readonly ILogger _logger;
private readonly ILogger<JobListener> _listenerLogger;
private IScheduler _scheduler;
public JobsHostedService(
IServiceProvider serviceProvider,
ILogger<JobsHostedService> logger,
ILogger<JobListener> listenerLogger)
{
_serviceProvider = serviceProvider;
_logger = logger;
_listenerLogger = listenerLogger;
}
public async Task StartAsync(CancellationToken cancellationToken)
{
var factory = new StdSchedulerFactory(new NameValueCollection
{
{ "quartz.serializer.type", "binary" }
});
_scheduler = await factory.GetScheduler(cancellationToken);
_scheduler.JobFactory = new JobFactory(_serviceProvider);
_scheduler.ListenerManager.AddJobListener(new JobListener(_listenerLogger),
GroupMatcher<JobKey>.AnyGroup());
await _scheduler.Start(cancellationToken);
var aliveJob = JobBuilder.Create<AliveJob>().Build();
var validateUsersJob = JobBuilder.Create<ValidateUsersJob>().Build();
var validateOrganizationsJob = JobBuilder.Create<ValidateOrganizationsJob>().Build();
var everyTopOfTheHourTrigger = TriggerBuilder.Create()
.StartNow()
.WithCronSchedule("0 0 * * * ?")
.Build();
var everyTopOfTheSixthHourTrigger = TriggerBuilder.Create()
.StartNow()
.WithCronSchedule("0 0 */6 * * ?")
.Build();
var everyTwelfthHourAndThirtyMinutesTrigger = TriggerBuilder.Create()
.StartNow()
.WithCronSchedule("0 30 */12 * * ?")
.Build();
await _scheduler.ScheduleJob(aliveJob, everyTopOfTheHourTrigger);
await _scheduler.ScheduleJob(validateUsersJob, everyTopOfTheSixthHourTrigger);
await _scheduler.ScheduleJob(validateOrganizationsJob, everyTwelfthHourAndThirtyMinutesTrigger);
}
public async Task StopAsync(CancellationToken cancellationToken)
{
await _scheduler?.Shutdown(cancellationToken);
}
public void Dispose()
{ }
public static void AddJobsServices(IServiceCollection services)
{
services.AddTransient<AliveJob>();
services.AddTransient<ValidateUsersJob>();
services.AddTransient<ValidateOrganizationsJob>();
}
}
}

34
src/Api/Jobs/ValidateOrganizationsJob.cs Normal file
View File

@ -0,0 +1,34 @@
using System;
using System.Threading.Tasks;
using Bit.Core.Services;
using Microsoft.Extensions.Logging;
using Quartz;
namespace Bit.Api.Jobs
{
public class ValidateOrganizationsJob : IJob
{
private readonly ILicensingService _licensingService;
private readonly ILogger<ValidateOrganizationsJob> _logger;
public ValidateOrganizationsJob(
ILicensingService licensingService,
ILogger<ValidateOrganizationsJob> logger)
{
_licensingService = licensingService;
_logger = logger;
}
public async Task Execute(IJobExecutionContext context)
{
try
{
await _licensingService.ValidateOrganizationsAsync();
}
catch(Exception e)
{
_logger.LogError(2, e, "Error performing {0}.", nameof(ValidateOrganizationsJob));
}
}
}
}

34
src/Api/Jobs/ValidateUsersJob.cs Normal file
View File

@ -0,0 +1,34 @@
using System;
using System.Threading.Tasks;
using Bit.Core.Services;
using Microsoft.Extensions.Logging;
using Quartz;
namespace Bit.Api.Jobs
{
public class ValidateUsersJob : IJob
{
private readonly ILicensingService _licensingService;
private readonly ILogger<ValidateUsersJob> _logger;
public ValidateUsersJob(
ILicensingService licensingService,
ILogger<ValidateUsersJob> logger)
{
_licensingService = licensingService;
_logger = logger;
}
public async Task Execute(IJobExecutionContext context)
{
try
{
await _licensingService.ValidateUsersAsync();
}
catch(Exception e)
{
_logger.LogError(2, e, "Error performing {0}.", nameof(ValidateUsersJob));
}
}
}
}

10
src/Api/Startup.cs
View File

@ -126,9 +126,15 @@ namespace Bit.Api
config.Filters.Add(new ModelStateValidationFilterAttribute());
}).AddJsonOptions(o => o.SerializerSettings.ContractResolver = new DefaultContractResolver());
// PDF generation
if(!globalSettings.SelfHosted)
if(globalSettings.SelfHosted)
{
// Jobs service
Jobs.JobsHostedService.AddJobsServices(services);
services.AddHostedService<Jobs.JobsHostedService>();
}
else
{
// PDF generation
services.AddJsReport(new jsreport.Local.LocalReporting()
.UseBinary(jsreport.Binary.JsReportBinary.GetBinary())
.AsUtility()

3
src/Api/build.sh
View File

@ -9,13 +9,10 @@ echo -e "\nBuilding app"
echo ".NET Core version $(dotnet --version)"
echo "Restore"
dotnet restore $DIR/Api.csproj
dotnet restore $DIR/../Jobs/Jobs.csproj
echo "Clean"
dotnet clean $DIR/Api.csproj -c "Release" -o $DIR/obj/Docker/publish/Api
dotnet clean $DIR/../Jobs/Jobs.csproj -c "Release" -o $DIR/obj/Docker/publish/Jobs
echo "Publish"
dotnet publish $DIR/Api.csproj -c "Release" -o $DIR/obj/Docker/publish/Api
dotnet publish $DIR/../Jobs/Jobs.csproj -c "Release" -o $DIR/obj/Docker/publish/Jobs
echo -e "\nBuilding docker image"
docker --version

7
src/Api/entrypoint.sh
View File

@ -29,19 +29,12 @@ mkhomedir_helper $USERNAME
# The rest...
touch /var/log/cron.log
chown $USERNAME:$GROUPNAME /var/log/cron.log
chown -R $USERNAME:$GROUPNAME /app
chown -R $USERNAME:$GROUPNAME /jobs
mkdir -p /etc/bitwarden/core
mkdir -p /etc/bitwarden/logs
mkdir -p /etc/bitwarden/ca-certificates
chown -R $USERNAME:$GROUPNAME /etc/bitwarden
# Sounds like gosu keeps env when switching, but of course cron does not
env > /etc/environment
cron
cp /etc/bitwarden/ca-certificates/*.crt /usr/local/share/ca-certificates/ >/dev/null 2>&1 \
&& update-ca-certificates

1
src/Core/Utilities/ServiceCollectionExtensions.cs
View File

@ -22,7 +22,6 @@ using SqlServerRepos = Bit.Core.Repositories.SqlServer;
using System.Threading.Tasks;
using TableStorageRepos = Bit.Core.Repositories.TableStorage;
using Microsoft.Extensions.DependencyInjection.Extensions;
using System.Collections.Generic;
namespace Bit.Core.Utilities
{

25
src/Jobs/Jobs.csproj
View File

@ -1,25 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
<RootNamespace>Bit.Jobs</RootNamespace>
<UserSecretsId>bitwarden-Jobs</UserSecretsId>
</PropertyGroup>
<ItemGroup>
<Content Include="crontab">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore" Version="2.1.2" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="2.1.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Core\Core.csproj" />
</ItemGroup>
</Project>

25
src/Jobs/NoopServer.cs
View File

@ -1,25 +0,0 @@
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Http.Features;
namespace Bit.Jobs
{
public class NoopServer : IServer
{
public IFeatureCollection Features => new FeatureCollection();
public void Dispose()
{ }
public Task StartAsync<TContext>(IHttpApplication<TContext> application, CancellationToken cancellationToken)
{
return Task.FromResult(0);
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.FromResult(0);
}
}
}

97
src/Jobs/Program.cs
View File

@ -1,97 +0,0 @@
using Bit.Core.Services;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
namespace Bit.Jobs
{
public class Program
{
private static ILicensingService _licensingService;
private static ILogger<Program> _logger;
public static void Main(string[] args)
{
var parameters = ParseParameters(args);
try
{
var host = new WebHostBuilder()
.UseContentRoot(parameters.ContainsKey("d") ? parameters["d"] : Directory.GetCurrentDirectory())
.UseStartup<Startup>()
.UseServer(new NoopServer())
.Build();
_logger = host.Services.GetRequiredService<ILogger<Program>>();
_licensingService = host.Services.GetRequiredService<ILicensingService>();
}
catch(Exception e)
{
if(_logger != null)
{
_logger.LogCritical(1, e, "Error while bootstrapping.");
}
throw e;
}
MainAsync(parameters).Wait();
}
private async static Task MainAsync(IDictionary<string, string> parameters)
{
if(!parameters.ContainsKey("j"))
{
return;
}
_logger.LogInformation("Starting job {0}.", parameters["j"]);
try
{
switch(parameters["j"])
{
case "validate-organizations":
await _licensingService.ValidateOrganizationsAsync();
break;
case "validate-users-premium":
await _licensingService.ValidateUsersAsync();
break;
case "refresh-licenses":
// TODO
break;
case "alive":
_logger.LogInformation(DateTime.UtcNow.ToString());
break;
default:
break;
}
}
catch(Exception e)
{
_logger.LogError(2, e, "Error performing job.");
throw e;
}
_logger.LogInformation("Finished job {0}.", parameters["j"]);
}
private static IDictionary<string, string> ParseParameters(string[] args)
{
var dict = new Dictionary<string, string>();
for(var i = 0; i < args.Length; i = i + 2)
{
if(!args[i].StartsWith("-"))
{
continue;
}
dict.Add(args[i].Substring(1), args[i + 1]);
}
return dict;
}
}
}

12
src/Jobs/Properties/launchSettings.json
View File

@ -1,12 +0,0 @@
{
"profiles": {
"Jobs": {
"commandName": "Project",
"launchBrowser": false,
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
},
"applicationUrl": "http://localhost:4409/"
}
}
}

74
src/Jobs/Startup.cs
View File

@ -1,74 +0,0 @@
using System;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Configuration;
using Bit.Core;
using Bit.Core.Utilities;
using Microsoft.Extensions.Logging;
using Microsoft.AspNetCore.Builder;
using Serilog.Events;
namespace Bit.Jobs
{
public class Startup
{
public Startup(IHostingEnvironment env)
{
var builder = new ConfigurationBuilder()
.SetBasePath(env.ContentRootPath)
.AddJsonFile("appsettings.json")
.AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true);
if(env.IsDevelopment())
{
builder.AddUserSecrets("bitwarden-Jobs");
}
builder.AddEnvironmentVariables();
Configuration = builder.Build();
Environment = env;
}
public IConfigurationRoot Configuration { get; private set; }
public IHostingEnvironment Environment { get; set; }
public void ConfigureServices(IServiceCollection services)
{
// Options
services.AddOptions();
// Settings
var globalSettings = services.AddGlobalSettingsServices(Configuration);
// Data Protection
services.AddCustomDataProtectionServices(Environment, globalSettings);
// Repositories
services.AddSqlServerRepositories(globalSettings);
// Context
services.AddScoped<CurrentContext>();
// Identity
services.AddCustomIdentityServices(globalSettings);
// Services
services.AddBaseServices();
services.AddDefaultServices(globalSettings);
}
public void Configure(
IApplicationBuilder app,
IHostingEnvironment env,
ILoggerFactory loggerFactory,
IApplicationLifetime appLifetime,
GlobalSettings globalSettings)
{
loggerFactory
.AddSerilog(app, env, appLifetime, globalSettings, e => e.Level >= LogEventLevel.Information)
.AddConsole()
.AddDebug();
}
}
}

17
src/Jobs/appsettings.Production.json
View File

@ -1,17 +0,0 @@
{
"globalSettings": {
"baseServiceUri": {
"vault": "https://vault.bitwarden.com",
"api": "https://api.bitwarden.com",
"identity": "https://identity.bitwarden.com",
"admin": "https://admin.bitwarden.com",
"internalAdmin": "https://admin.bitwarden.com",
"internalIdentity": "https://identity.bitwarden.com",
"internalApi": "https://api.bitwarden.com",
"internalVault": "https://vault.bitwarden.com"
},
"braintree": {
"production": true
}
}
}

58
src/Jobs/appsettings.json
View File

@ -1,58 +0,0 @@
{
"globalSettings": {
"selfHosted": false,
"siteName": "Bitwarden",
"projectName": "Jobs",
"stripeApiKey": "SECRET",
"baseServiceUri": {
"vault": "https://localhost:8080",
"api": "http://localhost:4000",
"identity": "http://localhost:33656",
"admin": "http://localhost:62911",
"internalAdmin": "http://localhost:62911",
"internalIdentity": "http://localhost:33656",
"internalApi": "http://localhost:4000",
"internalVault": "http://localhost:4001"
},
"sqlServer": {
"connectionString": "SECRET"
},
"mail": {
"sendGridApiKey": "SECRET",
"replyToEmail": "hello@bitwarden.com"
},
"identityServer": {
"certificateThumbprint": "SECRET"
},
"dataProtection": {
"certificateThumbprint": "SECRET"
},
"storage": {
"connectionString": "SECRET"
},
"documentDb": {
"uri": "SECRET",
"key": "SECRET"
},
"sentry": {
"dsn": "SECRET"
},
"notificationHub": {
"connectionString": "SECRET",
"hubName": "SECRET"
},
"yubico": {
"clientid": "SECRET",
"key": "SECRET"
},
"duo": {
"aKey": "SECRET"
},
"braintree": {
"production": false,
"merchantId": "SECRET",
"publicKey": "SECRET",
"privateKey": "SECRET"
}
}
}

5
src/Jobs/crontab
View File

@ -1,5 +0,0 @@
0 * * * * bitwarden dotnet /jobs/Jobs.dll -d /jobs -j alive >> /var/log/cron.log 2>&1
0 */6 * * * bitwarden dotnet /jobs/Jobs.dll -d /jobs -j validate-organizations >> /var/log/cron.log 2>&1
30 */12 * * * bitwarden dotnet /jobs/Jobs.dll -d /jobs -j validate-users-premium >> /var/log/cron.log 2>&1
# An empty line is required at the end of this file for a valid cron file.