From 07139684ced5223ce786280620c9d32b83764e7e Mon Sep 17 00:00:00 2001
From: cd1989 <chende@caicloud.io>
Date: Thu, 4 Apr 2019 22:17:29 +0800
Subject: [PATCH] Wait randomly before registry health checking

Signed-off-by: cd1989 <chende@caicloud.io>
---
 src/core/api/registry_test.go              | 2 +-
 src/core/main.go                           | 6 ++++--
 src/replication/ng/registry/healthcheck.go | 8 +++++++-
 src/replication/ng/replication.go          | 7 ++++++-
 src/replication/ng/replication_test.go     | 2 +-
 5 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/core/api/registry_test.go b/src/core/api/registry_test.go
index 7c5d5daad..70724f93f 100644
--- a/src/core/api/registry_test.go
+++ b/src/core/api/registry_test.go
@@ -44,7 +44,7 @@ type RegistrySuite struct {
 
 func (suite *RegistrySuite) SetupSuite() {
 	assert := assert.New(suite.T())
-	assert.Nil(ng.Init())
+	assert.Nil(ng.Init(make(chan struct{})))
 
 	suite.testAPI = newHarborAPI()
 	code, err := suite.testAPI.RegistryCreate(*admin, testRegistry)
diff --git a/src/core/main.go b/src/core/main.go
index 7b393cf15..9b2dbac3d 100644
--- a/src/core/main.go
+++ b/src/core/main.go
@@ -130,8 +130,10 @@ func main() {
 		}
 	}
 
-	if err := ng.Init(); err != nil {
-		log.Fatalf("failed to initialize replication: %v", err)
+	closing := make(chan struct{})
+	go gracefulShutdown(closing)
+	if err := ng.Init(closing); err != nil {
+		log.Fatalf("failed to init for replication: %v", err)
 	}
 
 	filter.Init()
diff --git a/src/replication/ng/registry/healthcheck.go b/src/replication/ng/registry/healthcheck.go
index 702c9c52b..0f26f2ac0 100644
--- a/src/replication/ng/registry/healthcheck.go
+++ b/src/replication/ng/registry/healthcheck.go
@@ -15,13 +15,14 @@
 package registry
 
 import (
+	"math/rand"
 	"time"
 
 	"github.com/goharbor/harbor/src/common/utils/log"
 )
 
 // MinInterval defines the minimum interval to check registries' health status.
-const MinInterval = time.Second * 3
+const MinInterval = time.Minute * 5
 
 // HealthChecker is used to regularly check all registries' health status and update
 // check result to database
@@ -48,6 +49,11 @@ func (c *HealthChecker) Run() {
 	if c.interval < MinInterval {
 		interval = MinInterval
 	}
+
+	// Wait some random time before starting health checking. If Harbor is deployed in HA mode
+	// with multiple instances, this will avoid instances check health in the same time.
+	<-time.After(time.Duration(rand.Int63n(int64(interval))))
+
 	ticker := time.NewTicker(interval)
 	log.Infof("Start regular health check for registries with interval %v", interval)
 	for {
diff --git a/src/replication/ng/replication.go b/src/replication/ng/replication.go
index 873e3d726..bf59d7c1c 100644
--- a/src/replication/ng/replication.go
+++ b/src/replication/ng/replication.go
@@ -17,6 +17,8 @@
 package ng
 
 import (
+	"time"
+
 	"github.com/goharbor/harbor/src/common/job"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	cfg "github.com/goharbor/harbor/src/core/config"
@@ -43,7 +45,7 @@ var (
 )
 
 // Init the global variables and configurations
-func Init() error {
+func Init(closing chan struct{}) error {
 	// init config
 	registryURL, err := cfg.RegistryURL()
 	if err != nil {
@@ -73,6 +75,9 @@ func Init() error {
 	// init event handler
 	EventHandler = event.NewHandler(PolicyCtl, RegistryMgr, OperationCtl)
 	log.Debug("the replication initialization completed")
+
+	// Start health checker for registries
+	go registry.NewHealthChecker(time.Minute*5, closing).Run()
 	return nil
 }
 
diff --git a/src/replication/ng/replication_test.go b/src/replication/ng/replication_test.go
index d876ef103..1e5793af9 100644
--- a/src/replication/ng/replication_test.go
+++ b/src/replication/ng/replication_test.go
@@ -36,7 +36,7 @@ func TestInit(t *testing.T) {
 	require.Nil(t, err)
 
 	config.InitWithSettings(nil)
-	err = Init()
+	err = Init(make(chan struct{}))
 	require.Nil(t, err)
 	assert.NotNil(t, PolicyCtl)
 	assert.NotNil(t, RegistryMgr)