From 57dd6c72944b380db0fad855ae352b94349e42c2 Mon Sep 17 00:00:00 2001 From: Oscar Hinton Date: Thu, 9 Sep 2021 18:13:48 +0200 Subject: [PATCH] Retry quartz initialization (#1570) --- src/Core/Jobs/BaseJobsHostedService.cs | 51 ++++++++++++++++++-------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/src/Core/Jobs/BaseJobsHostedService.cs b/src/Core/Jobs/BaseJobsHostedService.cs index 72758091a..57b7b60fd 100644 --- a/src/Core/Jobs/BaseJobsHostedService.cs +++ b/src/Core/Jobs/BaseJobsHostedService.cs @@ -15,6 +15,8 @@ namespace Bit.Core.Jobs { public abstract class BaseJobsHostedService : IHostedService, IDisposable { + private const int MaximumJobRetries = 10; + private readonly IServiceProvider _serviceProvider; private readonly ILogger _listenerLogger; protected readonly ILogger _logger; @@ -68,23 +70,42 @@ namespace Bit.Core.Jobs { foreach (var (job, trigger) in Jobs) { - var dupeT = await _scheduler.GetTrigger(trigger.Key); - if (dupeT != null) + for (var retry = 0; retry < MaximumJobRetries; retry++) { - await _scheduler.RescheduleJob(trigger.Key, trigger); + // There's a race condition when starting multiple containers simultaneously, retry until it succeeds.. + try + { + var dupeT = await _scheduler.GetTrigger(trigger.Key); + if (dupeT != null) + { + await _scheduler.RescheduleJob(trigger.Key, trigger); + } + + var jobDetail = JobBuilder.Create(job) + .WithIdentity(job.FullName) + .Build(); + + var dupeJ = await _scheduler.GetJobDetail(jobDetail.Key); + if (dupeJ != null) + { + await _scheduler.DeleteJob(jobDetail.Key); + } + + await _scheduler.ScheduleJob(jobDetail, trigger); + break; + } + catch (Exception e) + { + if (retry == MaximumJobRetries - 1) + { + throw new Exception("Job failed to start after 10 retries."); + } + + _logger.LogWarning($"Exception while trying to schedule job: {job.FullName}, {e}"); + var random = new Random(); + Thread.Sleep(random.Next(50, 250)); + } } - - var jobDetail = JobBuilder.Create(job) - .WithIdentity(job.FullName) - .Build(); - - var dupeJ = await _scheduler.GetJobDetail(jobDetail.Key); - if (dupeJ != null) - { - await _scheduler.DeleteJob(jobDetail.Key); - } - - await _scheduler.ScheduleJob(jobDetail, trigger); } }