From 29d0d51403787cffb6f7a1544bfa0b2d12982aed Mon Sep 17 00:00:00 2001 From: wangyan Date: Thu, 9 Aug 2018 04:04:35 -0700 Subject: [PATCH] Signed-off-by: wangyan Add clean registry cache to gc job To workaround the issue: https://github.com/docker/distribution/issues/2094 GC needs to clean cache before to call the docker reigstry api to delete blobs. Otherwise, the following docker push will not be performed as docker registry does not clean cache in GC, it thinks the image is still there, and the new blobs will be uploaded. --- Makefile | 2 +- make/common/templates/ui/env | 2 ++ make/prepare | 7 +++-- src/jobservice/job/impl/gc/job.go | 44 +++++++++++++++++++++++++++++-- src/ui/api/models/reg_gc.go | 12 +++++---- src/ui/api/reg_gc.go | 4 +++ 6 files changed, 61 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index f2f5ec680..c2c5bef7a 100644 --- a/Makefile +++ b/Makefile @@ -299,7 +299,7 @@ compile_golangimage: compile_clarity @$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME) @echo "Done." - @echo "compiling binary for harbor regsitry controller (golang image)..." + @echo "compiling binary for harbor registry controller (golang image)..." @$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME) @echo "Done." diff --git a/make/common/templates/ui/env b/make/common/templates/ui/env index e4d44bffd..fc9dedfb2 100644 --- a/make/common/templates/ui/env +++ b/make/common/templates/ui/env @@ -8,3 +8,5 @@ UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem _REDIS_URL=$redis_host:$redis_port,100,$redis_password SYNC_REGISTRY=false CHART_CACHE_DRIVER=$chart_cache_driver +_REDIS_URL_REG=$redis_url_reg + diff --git a/make/prepare b/make/prepare index 38f9f6111..0744bc96a 100755 --- a/make/prepare +++ b/make/prepare @@ -304,10 +304,13 @@ redis_db_index_chart = db_indexs[2] #redis://[arbitrary_username:password@]ipaddress:port/database_index redis_url_js = '' +redis_url_reg = '' if len(redis_password) > 0: redis_url_js = "redis://anonymous:%s@%s:%s/%s" % (redis_password, redis_host, redis_port, redis_db_index_js) + redis_url_reg = "redis://anonymous:%s@%s:%s/%s" % (redis_password, redis_host, redis_port, redis_db_index_reg) else: redis_url_js = "redis://%s:%s/%s" % (redis_host, redis_port, redis_db_index_js) + redis_url_reg = "redis://%s:%s/%s" % (redis_host, redis_port, redis_db_index_reg) if rcp.has_option("configuration", "skip_reload_env_pattern"): skip_reload_env_pattern = rcp.get("configuration", "skip_reload_env_pattern") @@ -463,8 +466,8 @@ render(os.path.join(templates_dir, "ui", "env"), redis_port=redis_port, redis_password=redis_password, adminserver_url = adminserver_url, - chart_cache_driver = chart_cache_driver - ) + chart_cache_driver = chart_cache_driver, + redis_url_reg = redis_url_reg) registry_config_file_ha = "config_ha.yml" registry_config_file = "config.yml" diff --git a/src/jobservice/job/impl/gc/job.go b/src/jobservice/job/impl/gc/job.go index 06871361f..18585cfc4 100644 --- a/src/jobservice/job/impl/gc/job.go +++ b/src/jobservice/job/impl/gc/job.go @@ -18,7 +18,9 @@ import ( "fmt" "net/http" "os" + "time" + "github.com/garyburd/redigo/redis" "github.com/vmware/harbor/src/common" common_http "github.com/vmware/harbor/src/common/http" "github.com/vmware/harbor/src/common/http/modifier/auth" @@ -29,6 +31,12 @@ import ( "github.com/vmware/harbor/src/registryctl/client" ) +const ( + dialConnectionTimeout = 30 * time.Second + dialReadTimeout = time.Minute + 10*time.Second + dialWriteTimeout = 10 * time.Second +) + // GarbageCollector is the struct to run registry's garbage collection type GarbageCollector struct { registryCtlClient client.Client @@ -36,6 +44,7 @@ type GarbageCollector struct { uiclient *common_http.Client UIURL string insecure bool + redisURL string } // MaxFails implements the interface in job/Interface @@ -55,7 +64,7 @@ func (gc *GarbageCollector) Validate(params map[string]interface{}) error { // Run implements the interface in job/Interface func (gc *GarbageCollector) Run(ctx env.JobContext, params map[string]interface{}) error { - if err := gc.init(ctx); err != nil { + if err := gc.init(ctx, params); err != nil { return err } if err := gc.readonly(true); err != nil { @@ -72,12 +81,15 @@ func (gc *GarbageCollector) Run(ctx env.JobContext, params map[string]interface{ gc.logger.Errorf("failed to get gc result: %v", err) return err } + if err := gc.cleanCache(); err != nil { + return err + } gc.logger.Infof("GC results: status: %t, message: %s, start: %s, end: %s.", gcr.Status, gcr.Msg, gcr.StartTime, gcr.EndTime) gc.logger.Infof("success to run gc in job.") return nil } -func (gc *GarbageCollector) init(ctx env.JobContext) error { +func (gc *GarbageCollector) init(ctx env.JobContext, params map[string]interface{}) error { registryctl.Init() gc.registryCtlClient = registryctl.RegistryCtlClient gc.logger = ctx.GetLogger() @@ -92,6 +104,7 @@ func (gc *GarbageCollector) init(ctx env.JobContext) error { } else { return fmt.Errorf(errTpl, common.UIURL) } + gc.redisURL = params["redis_url_reg"].(string) return nil } @@ -107,3 +120,30 @@ func (gc *GarbageCollector) readonly(switcher bool) error { gc.logger.Info("the readonly request has been sent successfully") return nil } + +// cleanCache is to clean the registry cache for GC. +// To do this is because the issue https://github.com/docker/distribution/issues/2094 +func (gc *GarbageCollector) cleanCache() error { + + con, err := redis.DialURL( + gc.redisURL, + redis.DialConnectTimeout(dialConnectionTimeout), + redis.DialReadTimeout(dialReadTimeout), + redis.DialWriteTimeout(dialWriteTimeout), + ) + + if err != nil { + gc.logger.Errorf("failed to connect to redis %v", err) + return err + } + defer con.Close() + + // clean all keys in registry redis DB. + _, err = con.Do("FLUSHDB") + if err != nil { + gc.logger.Errorf("failed to clean registry cache %v", err) + return err + } + + return nil +} diff --git a/src/ui/api/models/reg_gc.go b/src/ui/api/models/reg_gc.go index 7bb5e8e33..df568cc60 100644 --- a/src/ui/api/models/reg_gc.go +++ b/src/ui/api/models/reg_gc.go @@ -39,9 +39,10 @@ const ( // GCReq holds request information for admin job type GCReq struct { - Schedule *ScheduleParam `json:"schedule"` - Status string `json:"status"` - ID int64 `json:"id"` + Schedule *ScheduleParam `json:"schedule"` + Status string `json:"status"` + ID int64 `json:"id"` + Parameters map[string]interface{} `json:"parameters"` } //ScheduleParam defines the parameter of schedule trigger @@ -88,8 +89,9 @@ func (gr *GCReq) ToJob() (*models.JobData, error) { } jobData := &models.JobData{ - Name: job.ImageGC, - Metadata: metadata, + Name: job.ImageGC, + Parameters: gr.Parameters, + Metadata: metadata, StatusHook: fmt.Sprintf("%s/service/notifications/jobs/adminjob/%d", config.InternalUIURL(), gr.ID), } diff --git a/src/ui/api/reg_gc.go b/src/ui/api/reg_gc.go index cca8a704c..3f09d52ba 100644 --- a/src/ui/api/reg_gc.go +++ b/src/ui/api/reg_gc.go @@ -17,6 +17,7 @@ package api import ( "fmt" "net/http" + "os" "strconv" "github.com/vmware/harbor/src/common/dao" @@ -209,6 +210,9 @@ func (gc *GCAPI) submitJob(gr *models.GCReq) { return } gr.ID = id + gr.Parameters = map[string]interface{}{ + "redis_url_reg": os.Getenv("_REDIS_URL_REG"), + } job, err := gr.ToJob() if err != nil { gc.HandleInternalServerError(fmt.Sprintf("%v", err))