Wait randomly before registry health checking

Signed-off-by: cd1989 <chende@caicloud.io>

src/core/api/registry_test.go

@@ -44,7 +44,7 @@ type RegistrySuite struct {
 
 func (suite *RegistrySuite) SetupSuite() {
 	assert := assert.New(suite.T())
-	assert.Nil(ng.Init())
+	assert.Nil(ng.Init(make(chan struct{})))
 
 	suite.testAPI = newHarborAPI()
 	code, err := suite.testAPI.RegistryCreate(*admin, testRegistry)

src/core/main.go

@@ -130,8 +130,10 @@ func main() {
 		}
 	}
 
-	if err := ng.Init(); err != nil {
-		log.Fatalf("failed to initialize replication: %v", err)
+	closing := make(chan struct{})
+	go gracefulShutdown(closing)
+	if err := ng.Init(closing); err != nil {
+		log.Fatalf("failed to init for replication: %v", err)
 	}
 
 	filter.Init()

src/replication/ng/registry/healthcheck.go

@@ -15,13 +15,14 @@
 package registry
 
 import (
+	"math/rand"
 	"time"
 
 	"github.com/goharbor/harbor/src/common/utils/log"
 )
 
 // MinInterval defines the minimum interval to check registries' health status.
-const MinInterval = time.Second * 3
+const MinInterval = time.Minute * 5
 
 // HealthChecker is used to regularly check all registries' health status and update
 // check result to database
@@ -48,6 +49,11 @@ func (c *HealthChecker) Run() {
 	if c.interval < MinInterval {
 		interval = MinInterval
 	}
+
+	// Wait some random time before starting health checking. If Harbor is deployed in HA mode
+	// with multiple instances, this will avoid instances check health in the same time.
+	<-time.After(time.Duration(rand.Int63n(int64(interval))))
+
 	ticker := time.NewTicker(interval)
 	log.Infof("Start regular health check for registries with interval %v", interval)
 	for {

src/replication/ng/replication.go

@@ -17,6 +17,8 @@
 package ng
 
 import (
+	"time"
+
 	"github.com/goharbor/harbor/src/common/job"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	cfg "github.com/goharbor/harbor/src/core/config"
@@ -43,7 +45,7 @@ var (
 )
 
 // Init the global variables and configurations
-func Init() error {
+func Init(closing chan struct{}) error {
 	// init config
 	registryURL, err := cfg.RegistryURL()
 	if err != nil {
@@ -73,6 +75,9 @@ func Init() error {
 	// init event handler
 	EventHandler = event.NewHandler(PolicyCtl, RegistryMgr, OperationCtl)
 	log.Debug("the replication initialization completed")
+
+	// Start health checker for registries
+	go registry.NewHealthChecker(time.Minute*5, closing).Run()
 	return nil
 }
 

src/replication/ng/replication_test.go

@@ -36,7 +36,7 @@ func TestInit(t *testing.T) {
 	require.Nil(t, err)
 
 	config.InitWithSettings(nil)
-	err = Init()
+	err = Init(make(chan struct{}))
 	require.Nil(t, err)
 	assert.NotNil(t, PolicyCtl)
 	assert.NotNil(t, RegistryMgr)