From b9d6108624001edfbe4a7d2dba28b753c487d6bd Mon Sep 17 00:00:00 2001 From: wang yan Date: Wed, 14 Aug 2019 12:39:23 +0800 Subject: [PATCH] add ping for adapter to wait for service ready Signed-off-by: wang yan --- src/core/api/health.go | 31 +++++++------- src/core/api/health_test.go | 16 ++++---- src/core/api/quota/chart/chart.go | 12 ++++-- src/core/api/quota/migrator.go | 54 +++++++++++++++++++++++-- src/core/api/quota/registry/registry.go | 5 +++ 5 files changed, 89 insertions(+), 29 deletions(-) diff --git a/src/core/api/health.go b/src/core/api/health.go index 6d5a890d1..0d4ef2cac 100644 --- a/src/core/api/health.go +++ b/src/core/api/health.go @@ -34,8 +34,9 @@ import ( ) var ( - timeout = 60 * time.Second - healthCheckerRegistry = map[string]health.Checker{} + timeout = 60 * time.Second + // HealthCheckerRegistry ... + HealthCheckerRegistry = map[string]health.Checker{} ) type overallHealthStatus struct { @@ -67,11 +68,11 @@ type HealthAPI struct { func (h *HealthAPI) CheckHealth() { var isHealthy healthy = true components := []*componentHealthStatus{} - c := make(chan *componentHealthStatus, len(healthCheckerRegistry)) - for name, checker := range healthCheckerRegistry { + c := make(chan *componentHealthStatus, len(HealthCheckerRegistry)) + for name, checker := range HealthCheckerRegistry { go check(name, checker, timeout, c) } - for i := 0; i < len(healthCheckerRegistry); i++ { + for i := 0; i < len(HealthCheckerRegistry); i++ { componentStatus := <-c if len(componentStatus.Error) != 0 { isHealthy = false @@ -290,21 +291,21 @@ func redisHealthChecker() health.Checker { } func registerHealthCheckers() { - healthCheckerRegistry["core"] = coreHealthChecker() - healthCheckerRegistry["portal"] = portalHealthChecker() - healthCheckerRegistry["jobservice"] = jobserviceHealthChecker() - healthCheckerRegistry["registry"] = registryHealthChecker() - healthCheckerRegistry["registryctl"] = registryCtlHealthChecker() - healthCheckerRegistry["database"] = databaseHealthChecker() - healthCheckerRegistry["redis"] = redisHealthChecker() + HealthCheckerRegistry["core"] = coreHealthChecker() + HealthCheckerRegistry["portal"] = portalHealthChecker() + HealthCheckerRegistry["jobservice"] = jobserviceHealthChecker() + HealthCheckerRegistry["registry"] = registryHealthChecker() + HealthCheckerRegistry["registryctl"] = registryCtlHealthChecker() + HealthCheckerRegistry["database"] = databaseHealthChecker() + HealthCheckerRegistry["redis"] = redisHealthChecker() if config.WithChartMuseum() { - healthCheckerRegistry["chartmuseum"] = chartmuseumHealthChecker() + HealthCheckerRegistry["chartmuseum"] = chartmuseumHealthChecker() } if config.WithClair() { - healthCheckerRegistry["clair"] = clairHealthChecker() + HealthCheckerRegistry["clair"] = clairHealthChecker() } if config.WithNotary() { - healthCheckerRegistry["notary"] = notaryHealthChecker() + HealthCheckerRegistry["notary"] = notaryHealthChecker() } } diff --git a/src/core/api/health_test.go b/src/core/api/health_test.go index 8426a74b1..c98d021b5 100644 --- a/src/core/api/health_test.go +++ b/src/core/api/health_test.go @@ -92,9 +92,9 @@ func fakeHealthChecker(healthy bool) health.Checker { } func TestCheckHealth(t *testing.T) { // component01: healthy, component02: healthy => status: healthy - healthCheckerRegistry = map[string]health.Checker{} - healthCheckerRegistry["component01"] = fakeHealthChecker(true) - healthCheckerRegistry["component02"] = fakeHealthChecker(true) + HealthCheckerRegistry = map[string]health.Checker{} + HealthCheckerRegistry["component01"] = fakeHealthChecker(true) + HealthCheckerRegistry["component02"] = fakeHealthChecker(true) status := map[string]interface{}{} err := handleAndParse(&testingRequest{ method: http.MethodGet, @@ -104,9 +104,9 @@ func TestCheckHealth(t *testing.T) { assert.Equal(t, "healthy", status["status"].(string)) // component01: healthy, component02: unhealthy => status: unhealthy - healthCheckerRegistry = map[string]health.Checker{} - healthCheckerRegistry["component01"] = fakeHealthChecker(true) - healthCheckerRegistry["component02"] = fakeHealthChecker(false) + HealthCheckerRegistry = map[string]health.Checker{} + HealthCheckerRegistry["component01"] = fakeHealthChecker(true) + HealthCheckerRegistry["component02"] = fakeHealthChecker(false) status = map[string]interface{}{} err = handleAndParse(&testingRequest{ method: http.MethodGet, @@ -128,7 +128,7 @@ func TestDatabaseHealthChecker(t *testing.T) { } func TestRegisterHealthCheckers(t *testing.T) { - healthCheckerRegistry = map[string]health.Checker{} + HealthCheckerRegistry = map[string]health.Checker{} registerHealthCheckers() - assert.NotNil(t, healthCheckerRegistry["core"]) + assert.NotNil(t, HealthCheckerRegistry["core"]) } diff --git a/src/core/api/quota/chart/chart.go b/src/core/api/quota/chart/chart.go index 31dbb42ec..f3ebc1f11 100644 --- a/src/core/api/quota/chart/chart.go +++ b/src/core/api/quota/chart/chart.go @@ -21,6 +21,7 @@ import ( "github.com/goharbor/harbor/src/common/models" common_quota "github.com/goharbor/harbor/src/common/quota" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/api" quota "github.com/goharbor/harbor/src/core/api/quota" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/promgr" @@ -49,6 +50,11 @@ var ( controllerOnce sync.Once ) +// Ping ... +func (rm *Migrator) Ping() error { + return api.HealthCheckerRegistry["chartmuseum"].Check() +} + // Dump ... // Depends on DB to dump chart data, as chart cannot get all of namespaces. func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) { @@ -99,8 +105,6 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) { defer wg.Done() var repos []quota.RepoData - var afs []*models.Artifact - ctr, err := chartController() if err != nil { errChan <- err @@ -115,6 +119,7 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) { // repo for _, chart := range chartInfo { + var afs []*models.Artifact chartVersions, err := ctr.GetChart(project.Name, chart.Name) if err != nil { errChan <- err @@ -171,7 +176,8 @@ func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, e proUsage := quota.ProjectUsage{ Project: project.Name, Used: common_quota.ResourceList{ - common_quota.ResourceCount: count, + common_quota.ResourceCount: count, + common_quota.ResourceStorage: 0, }, } pros = append(pros, proUsage) diff --git a/src/core/api/quota/migrator.go b/src/core/api/quota/migrator.go index b7f36b61a..bfd2fc164 100644 --- a/src/core/api/quota/migrator.go +++ b/src/core/api/quota/migrator.go @@ -21,11 +21,15 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/promgr" + "github.com/goharbor/harbor/src/pkg/types" "strconv" ) // QuotaMigrator ... type QuotaMigrator interface { + // Ping validates and wait for backend service ready. + Ping() error + // Dump exports all data from backend service, registry, chartmuseum Dump() ([]ProjectInfo, error) @@ -74,6 +78,7 @@ func Register(name string, adapter Instance) { // Sync ... func Sync(pm promgr.ProjectManager, populate bool) error { + totalUsage := make(map[string][]ProjectUsage) for name, instanceFunc := range adapters { if !config.WithChartMuseum() { if name == "chart" { @@ -81,6 +86,9 @@ func Sync(pm promgr.ProjectManager, populate bool) error { } } adapter := instanceFunc(pm) + if err := adapter.Ping(); err != nil { + return err + } data, err := adapter.Dump() if err != nil { return err @@ -89,18 +97,58 @@ func Sync(pm promgr.ProjectManager, populate bool) error { if err != nil { return err } - if err := ensureQuota(usage); err != nil { - return err - } + totalUsage[name] = usage if populate { if err := adapter.Persist(data); err != nil { return err } } } + merged := mergeUsage(totalUsage) + if err := ensureQuota(merged); err != nil { + return err + } return nil } +// mergeUsage merges the usage of adapters +func mergeUsage(total map[string][]ProjectUsage) []ProjectUsage { + if !config.WithChartMuseum() { + return total["registry"] + } + regUsgs := total["registry"] + chartUsgs := total["chart"] + + var mergedUsage []ProjectUsage + temp := make(map[string]quota.ResourceList) + + for _, regUsg := range regUsgs { + _, exist := temp[regUsg.Project] + if !exist { + temp[regUsg.Project] = regUsg.Used + mergedUsage = append(mergedUsage, ProjectUsage{ + Project: regUsg.Project, + Used: regUsg.Used, + }) + } + } + for _, chartUsg := range chartUsgs { + var usedTemp quota.ResourceList + _, exist := temp[chartUsg.Project] + if !exist { + usedTemp = chartUsg.Used + } else { + usedTemp = types.Add(temp[chartUsg.Project], chartUsg.Used) + } + temp[chartUsg.Project] = usedTemp + mergedUsage = append(mergedUsage, ProjectUsage{ + Project: chartUsg.Project, + Used: usedTemp, + }) + } + return mergedUsage +} + // ensureQuota updates the quota and quota usage in the data base. func ensureQuota(usages []ProjectUsage) error { var pid int64 diff --git a/src/core/api/quota/registry/registry.go b/src/core/api/quota/registry/registry.go index 5f69fb710..18a7f87c0 100644 --- a/src/core/api/quota/registry/registry.go +++ b/src/core/api/quota/registry/registry.go @@ -46,6 +46,11 @@ func NewRegistryMigrator(pm promgr.ProjectManager) quota.QuotaMigrator { return &migrator } +// Ping ... +func (rm *Migrator) Ping() error { + return api.HealthCheckerRegistry["registry"].Check() +} + // Dump ... func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) { var (