Upstream/bitwarden-server
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2025-02-18 02:11:22 +01:00
Code Issues Projects Releases Wiki Activity

Retry quartz initialization (#1570)

Browse Source
This commit is contained in:
Oscar Hinton 2021-09-09 18:13:48 +02:00 committed by GitHub
parent d977d27d7d
commit 57dd6c7294
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 36 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
src/Core/Jobs/BaseJobsHostedService.cs
View File

@ -15,6 +15,8 @@ namespace Bit.Core.Jobs
{ {
public abstract class BaseJobsHostedService : IHostedService, IDisposable public abstract class BaseJobsHostedService : IHostedService, IDisposable
{ {
private const int MaximumJobRetries = 10;
private readonly IServiceProvider _serviceProvider; private readonly IServiceProvider _serviceProvider;
private readonly ILogger<JobListener> _listenerLogger; private readonly ILogger<JobListener> _listenerLogger;
protected readonly ILogger _logger; protected readonly ILogger _logger;
@ -68,23 +70,42 @@ namespace Bit.Core.Jobs
{ {
foreach (var (job, trigger) in Jobs) foreach (var (job, trigger) in Jobs)
{ {
var dupeT = await _scheduler.GetTrigger(trigger.Key); for (var retry = 0; retry < MaximumJobRetries; retry++)
if (dupeT != null)
{ {
await _scheduler.RescheduleJob(trigger.Key, trigger); // There's a race condition when starting multiple containers simultaneously, retry until it succeeds..
try
{
var dupeT = await _scheduler.GetTrigger(trigger.Key);
if (dupeT != null)
{
await _scheduler.RescheduleJob(trigger.Key, trigger);
}
var jobDetail = JobBuilder.Create(job)
.WithIdentity(job.FullName)
.Build();
var dupeJ = await _scheduler.GetJobDetail(jobDetail.Key);
if (dupeJ != null)
{
await _scheduler.DeleteJob(jobDetail.Key);
}
await _scheduler.ScheduleJob(jobDetail, trigger);
break;
}
catch (Exception e)
{
if (retry == MaximumJobRetries - 1)
{
throw new Exception("Job failed to start after 10 retries.");
}
_logger.LogWarning($"Exception while trying to schedule job: {job.FullName}, {e}");
var random = new Random();
Thread.Sleep(random.Next(50, 250));
}
} }
var jobDetail = JobBuilder.Create(job)
.WithIdentity(job.FullName)
.Build();
var dupeJ = await _scheduler.GetJobDetail(jobDetail.Key);
if (dupeJ != null)
{
await _scheduler.DeleteJob(jobDetail.Key);
}
await _scheduler.ScheduleJob(jobDetail, trigger);
} }
} }