Merge pull request #5592 from wy65701436/add-clean-redis-in-gc

Add clean registry cache to gc job

Makefile

@@ -299,7 +299,7 @@ compile_golangimage: compile_clarity
 	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME)
 	@echo "Done."
 
-	@echo "compiling binary for harbor regsitry controller (golang image)..."
+	@echo "compiling binary for harbor registry controller (golang image)..."
 	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
 	@echo "Done."
 

make/common/templates/ui/env

@@ -8,3 +8,5 @@ UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem
 _REDIS_URL=$redis_host:$redis_port,100,$redis_password
 SYNC_REGISTRY=false
 CHART_CACHE_DRIVER=$chart_cache_driver
+_REDIS_URL_REG=$redis_url_reg
+

make/prepare

@@ -304,10 +304,13 @@ redis_db_index_chart = db_indexs[2]
 
 #redis://[arbitrary_username:password@]ipaddress:port/database_index
 redis_url_js = ''
+redis_url_reg = ''
 if len(redis_password) > 0:
     redis_url_js = "redis://anonymous:%s@%s:%s/%s" % (redis_password, redis_host, redis_port, redis_db_index_js)
+    redis_url_reg = "redis://anonymous:%s@%s:%s/%s" % (redis_password, redis_host, redis_port, redis_db_index_reg)
 else:
     redis_url_js = "redis://%s:%s/%s" % (redis_host, redis_port, redis_db_index_js)
+    redis_url_reg = "redis://%s:%s/%s" % (redis_host, redis_port, redis_db_index_reg)
 
 if rcp.has_option("configuration", "skip_reload_env_pattern"):
     skip_reload_env_pattern = rcp.get("configuration", "skip_reload_env_pattern")
@@ -463,8 +466,8 @@ render(os.path.join(templates_dir, "ui", "env"),
         redis_port=redis_port,
         redis_password=redis_password,
         adminserver_url = adminserver_url,
-        chart_cache_driver = chart_cache_driver
-        )
+        chart_cache_driver = chart_cache_driver,
+        redis_url_reg = redis_url_reg)
 
 registry_config_file_ha = "config_ha.yml"
 registry_config_file = "config.yml"

src/jobservice/job/impl/gc/job.go

@@ -18,7 +18,9 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"time"
 
+	"github.com/garyburd/redigo/redis"
 	"github.com/vmware/harbor/src/common"
 	common_http "github.com/vmware/harbor/src/common/http"
 	"github.com/vmware/harbor/src/common/http/modifier/auth"
@@ -29,6 +31,12 @@ import (
 	"github.com/vmware/harbor/src/registryctl/client"
 )
 
+const (
+	dialConnectionTimeout = 30 * time.Second
+	dialReadTimeout       = time.Minute + 10*time.Second
+	dialWriteTimeout      = 10 * time.Second
+)
+
 // GarbageCollector is the struct to run registry's garbage collection
 type GarbageCollector struct {
 	registryCtlClient client.Client
@@ -36,6 +44,7 @@ type GarbageCollector struct {
 	uiclient          *common_http.Client
 	UIURL             string
 	insecure          bool
+	redisURL          string
 }
 
 // MaxFails implements the interface in job/Interface
@@ -55,7 +64,7 @@ func (gc *GarbageCollector) Validate(params map[string]interface{}) error {
 
 // Run implements the interface in job/Interface
 func (gc *GarbageCollector) Run(ctx env.JobContext, params map[string]interface{}) error {
-	if err := gc.init(ctx); err != nil {
+	if err := gc.init(ctx, params); err != nil {
 		return err
 	}
 	if err := gc.readonly(true); err != nil {
@@ -72,12 +81,15 @@ func (gc *GarbageCollector) Run(ctx env.JobContext, params map[string]interface{
 		gc.logger.Errorf("failed to get gc result: %v", err)
 		return err
 	}
+	if err := gc.cleanCache(); err != nil {
+		return err
+	}
 	gc.logger.Infof("GC results: status: %t, message: %s, start: %s, end: %s.", gcr.Status, gcr.Msg, gcr.StartTime, gcr.EndTime)
 	gc.logger.Infof("success to run gc in job.")
 	return nil
 }
 
-func (gc *GarbageCollector) init(ctx env.JobContext) error {
+func (gc *GarbageCollector) init(ctx env.JobContext, params map[string]interface{}) error {
 	registryctl.Init()
 	gc.registryCtlClient = registryctl.RegistryCtlClient
 	gc.logger = ctx.GetLogger()
@@ -92,6 +104,7 @@ func (gc *GarbageCollector) init(ctx env.JobContext) error {
 	} else {
 		return fmt.Errorf(errTpl, common.UIURL)
 	}
+	gc.redisURL = params["redis_url_reg"].(string)
 	return nil
 }
 
@@ -107,3 +120,30 @@ func (gc *GarbageCollector) readonly(switcher bool) error {
 	gc.logger.Info("the readonly request has been sent successfully")
 	return nil
 }
+
+// cleanCache is to clean the registry cache for GC.
+// To do this is because the issue https://github.com/docker/distribution/issues/2094
+func (gc *GarbageCollector) cleanCache() error {
+
+	con, err := redis.DialURL(
+		gc.redisURL,
+		redis.DialConnectTimeout(dialConnectionTimeout),
+		redis.DialReadTimeout(dialReadTimeout),
+		redis.DialWriteTimeout(dialWriteTimeout),
+	)
+
+	if err != nil {
+		gc.logger.Errorf("failed to connect to redis %v", err)
+		return err
+	}
+	defer con.Close()
+
+	// clean all keys in registry redis DB.
+	_, err = con.Do("FLUSHDB")
+	if err != nil {
+		gc.logger.Errorf("failed to clean registry cache %v", err)
+		return err
+	}
+
+	return nil
+}

src/ui/api/models/reg_gc.go

@@ -39,9 +39,10 @@ const (
 
 // GCReq holds request information for admin job
 type GCReq struct {
-	Schedule *ScheduleParam `json:"schedule"`
-	Status   string         `json:"status"`
-	ID       int64          `json:"id"`
+	Schedule   *ScheduleParam         `json:"schedule"`
+	Status     string                 `json:"status"`
+	ID         int64                  `json:"id"`
+	Parameters map[string]interface{} `json:"parameters"`
 }
 
 //ScheduleParam defines the parameter of schedule trigger
@@ -88,8 +89,9 @@ func (gr *GCReq) ToJob() (*models.JobData, error) {
 	}
 
 	jobData := &models.JobData{
-		Name:     job.ImageGC,
-		Metadata: metadata,
+		Name:       job.ImageGC,
+		Parameters: gr.Parameters,
+		Metadata:   metadata,
 		StatusHook: fmt.Sprintf("%s/service/notifications/jobs/adminjob/%d",
 			config.InternalUIURL(), gr.ID),
 	}

src/ui/api/reg_gc.go

@@ -17,6 +17,7 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"os"
 	"strconv"
 
 	"github.com/vmware/harbor/src/common/dao"
@@ -209,6 +210,9 @@ func (gc *GCAPI) submitJob(gr *models.GCReq) {
 		return
 	}
 	gr.ID = id
+	gr.Parameters = map[string]interface{}{
+		"redis_url_reg": os.Getenv("_REDIS_URL_REG"),
+	}
 	job, err := gr.ToJob()
 	if err != nil {
 		gc.HandleInternalServerError(fmt.Sprintf("%v", err))