From 566169f59a3f12be94f6c70f81b071cc0d63f58a Mon Sep 17 00:00:00 2001 From: wang yan Date: Thu, 25 Oct 2018 16:41:53 +0800 Subject: [PATCH 1/2] Fix gc jobs issues. This commit is to fix two problems in the gc job, 1, in order to support redis cluseter, gc job should delete the keys only for docker reigstry instead of all keys. It because that in the redis cluster, only db 0 is supported. 2, to restore the real status of read only not just to set it to false. Signed-off-by: wang yan --- src/jobservice/job/impl/gc/job.go | 73 +++++++++++++++++++++++++++---- src/registryctl/api/registry.go | 2 + 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/src/jobservice/job/impl/gc/job.go b/src/jobservice/job/impl/gc/job.go index bc2f3fba58..87a53618d7 100644 --- a/src/jobservice/job/impl/gc/job.go +++ b/src/jobservice/job/impl/gc/job.go @@ -25,6 +25,7 @@ import ( common_http "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/http/modifier/auth" "github.com/goharbor/harbor/src/common/registryctl" + "github.com/goharbor/harbor/src/common/utils" reg "github.com/goharbor/harbor/src/common/utils/registry" "github.com/goharbor/harbor/src/jobservice/env" "github.com/goharbor/harbor/src/jobservice/logger" @@ -35,13 +36,15 @@ const ( dialConnectionTimeout = 30 * time.Second dialReadTimeout = time.Minute + 10*time.Second dialWriteTimeout = 10 * time.Second + blobPrefix = "blobs::*" + repoPrefix = "repository::*" ) // GarbageCollector is the struct to run registry's garbage collection type GarbageCollector struct { registryCtlClient client.Client logger logger.Interface - uiclient *common_http.Client + coreclient *common_http.Client CoreURL string insecure bool redisURL string @@ -67,10 +70,16 @@ func (gc *GarbageCollector) Run(ctx env.JobContext, params map[string]interface{ if err := gc.init(ctx, params); err != nil { return err } - if err := gc.readonly(true); err != nil { + readOnlyCur, err := gc.getReadOnly() + if err != nil { return err } - defer gc.readonly(false) + if readOnlyCur != true { + if err := gc.setReadOnly(true); err != nil { + return err + } + } + defer gc.setReadOnly(readOnlyCur) if err := gc.registryCtlClient.Health(); err != nil { gc.logger.Errorf("failed to start gc as registry controller is unreachable: %v", err) return err @@ -95,7 +104,7 @@ func (gc *GarbageCollector) init(ctx env.JobContext, params map[string]interface gc.logger = ctx.GetLogger() cred := auth.NewSecretAuthorizer(os.Getenv("JOBSERVICE_SECRET")) gc.insecure = false - gc.uiclient = common_http.NewClient(&http.Client{ + gc.coreclient = common_http.NewClient(&http.Client{ Transport: reg.GetHTTPTransport(gc.insecure), }, cred) errTpl := "Failed to get required property: %s" @@ -108,8 +117,16 @@ func (gc *GarbageCollector) init(ctx env.JobContext, params map[string]interface return nil } -func (gc *GarbageCollector) readonly(switcher bool) error { - if err := gc.uiclient.Put(fmt.Sprintf("%s/api/configurations", gc.CoreURL), struct { +func (gc *GarbageCollector) getReadOnly() (bool, error) { + cfgs := map[string]interface{}{} + if err := gc.coreclient.Get(fmt.Sprintf("%s/api/configs", gc.CoreURL), &cfgs); err != nil { + return false, err + } + return utils.SafeCastBool(cfgs[common.ReadOnly]), nil +} + +func (gc *GarbageCollector) setReadOnly(switcher bool) error { + if err := gc.coreclient.Put(fmt.Sprintf("%s/api/configurations", gc.CoreURL), struct { ReadOnly bool `json:"read_only"` }{ ReadOnly: switcher, @@ -139,11 +156,51 @@ func (gc *GarbageCollector) cleanCache() error { defer con.Close() // clean all keys in registry redis DB. - _, err = con.Do("FLUSHDB") + + // sample of keys in registry redis: + // 1) "blobs::sha256:1a6fd470b9ce10849be79e99529a88371dff60c60aab424c077007f6979b4812" + // 2) "repository::library/hello-world::blobs::sha256:4ab4c602aa5eed5528a6620ff18a1dc4faef0e1ab3a5eddeddb410714478c67f" + err = delKeys(con, blobPrefix) if err != nil { - gc.logger.Errorf("failed to clean registry cache %v", err) + gc.logger.Errorf("failed to clean registry cache %v, pattern blobs::*", err) + return err + } + err = delKeys(con, repoPrefix) + if err != nil { + gc.logger.Errorf("failed to clean registry cache %v, pattern repository::*", err) return err } return nil } + +func delKeys(con redis.Conn, pattern string) error { + iter := 0 + keys := []string{} + for { + arr, err := redis.Values(con.Do("SCAN", iter, "MATCH", pattern)) + if err != nil { + return fmt.Errorf("error retrieving '%s' keys", pattern) + } + iter, err := redis.Int(arr[0], nil) + if err != nil { + return fmt.Errorf("unexpected type for Int, got type %T", err) + } + k, err := redis.Strings(arr[1], nil) + if err != nil { + return fmt.Errorf("converts an array command reply to a []string %v", err) + } + keys = append(keys, k...) + + if iter == 0 { + break + } + } + for _, key := range keys { + _, err := con.Do("DEL", key) + if err != nil { + return fmt.Errorf("failed to clean registry cache %v", err) + } + } + return nil +} diff --git a/src/registryctl/api/registry.go b/src/registryctl/api/registry.go index 8104c741d8..d834327d89 100644 --- a/src/registryctl/api/registry.go +++ b/src/registryctl/api/registry.go @@ -44,6 +44,7 @@ func StartGC(w http.ResponseWriter, r *http.Request) { cmd.Stderr = &errBuf start := time.Now() + log.Debugf("Start to execute garbage collection...") if err := cmd.Run(); err != nil { log.Errorf("Fail to execute GC: %v, command err: %s", err, errBuf.String()) handleInternalServerError(w) @@ -55,4 +56,5 @@ func StartGC(w http.ResponseWriter, r *http.Request) { log.Errorf("failed to write response: %v", err) return } + log.Debugf("Successful to execute garbage collection...") } From 903d3906ab1efb1a5aaa2660c6914cc77368bcd8 Mon Sep 17 00:00:00 2001 From: wang yan Date: Mon, 29 Oct 2018 15:27:53 +0800 Subject: [PATCH 2/2] Fix per comments by wk Signed-off-by: wang yan --- src/jobservice/job/impl/gc/job.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jobservice/job/impl/gc/job.go b/src/jobservice/job/impl/gc/job.go index 87a53618d7..7ca6689acd 100644 --- a/src/jobservice/job/impl/gc/job.go +++ b/src/jobservice/job/impl/gc/job.go @@ -78,8 +78,8 @@ func (gc *GarbageCollector) Run(ctx env.JobContext, params map[string]interface{ if err := gc.setReadOnly(true); err != nil { return err } + defer gc.setReadOnly(readOnlyCur) } - defer gc.setReadOnly(readOnlyCur) if err := gc.registryCtlClient.Health(); err != nil { gc.logger.Errorf("failed to start gc as registry controller is unreachable: %v", err) return err @@ -182,7 +182,7 @@ func delKeys(con redis.Conn, pattern string) error { if err != nil { return fmt.Errorf("error retrieving '%s' keys", pattern) } - iter, err := redis.Int(arr[0], nil) + iter, err = redis.Int(arr[0], nil) if err != nil { return fmt.Errorf("unexpected type for Int, got type %T", err) }