Signed-off-by: wangyan <wangyan@vmware.com>

Add clean registry cache to gc job

To workaround the issue: https://github.com/docker/distribution/issues/2094
GC needs to clean cache before to call the docker reigstry api to delete blobs.
Otherwise, the following docker push will not be performed as docker registry
does not clean cache in GC, it thinks the image is still there, and the new
blobs will be uploaded.

Makefile

@@ -299,7 +299,7 @@ compile_golangimage: compile_clarity
 	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME)
 	@echo "Done."
 
-	@echo "compiling binary for harbor regsitry controller (golang image)..."
+	@echo "compiling binary for harbor registry controller (golang image)..."
 	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
 	@echo "Done."
 

make/common/templates/ui/env

@@ -8,3 +8,5 @@ UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem
 _REDIS_URL=$redis_host:$redis_port,100,$redis_password
 SYNC_REGISTRY=false
 CHART_CACHE_DRIVER=$chart_cache_driver
+_REDIS_URL_REG=$redis_url_reg
+

make/prepare

@@ -304,10 +304,13 @@ redis_db_index_chart = db_indexs[2]
 
 #redis://[arbitrary_username:password@]ipaddress:port/database_index
 redis_url_js = ''
+redis_url_reg = ''
 if len(redis_password) > 0:
     redis_url_js = "redis://anonymous:%s@%s:%s/%s" % (redis_password, redis_host, redis_port, redis_db_index_js)
+    redis_url_reg = "redis://anonymous:%s@%s:%s/%s" % (redis_password, redis_host, redis_port, redis_db_index_reg)
 else:
     redis_url_js = "redis://%s:%s/%s" % (redis_host, redis_port, redis_db_index_js)
+    redis_url_reg = "redis://%s:%s/%s" % (redis_host, redis_port, redis_db_index_reg)
 
 if rcp.has_option("configuration", "skip_reload_env_pattern"):
     skip_reload_env_pattern = rcp.get("configuration", "skip_reload_env_pattern")
@@ -463,8 +466,8 @@ render(os.path.join(templates_dir, "ui", "env"),
         redis_port=redis_port,
         redis_password=redis_password,
         adminserver_url = adminserver_url,
-        chart_cache_driver = chart_cache_driver
-        )
+        chart_cache_driver = chart_cache_driver,
+        redis_url_reg = redis_url_reg)
 
 registry_config_file_ha = "config_ha.yml"
 registry_config_file = "config.yml"

src/jobservice/job/impl/gc/job.go

@@ -18,7 +18,9 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"time"
 
+	"github.com/garyburd/redigo/redis"
 	"github.com/vmware/harbor/src/common"
 	common_http "github.com/vmware/harbor/src/common/http"
 	"github.com/vmware/harbor/src/common/http/modifier/auth"
@@ -29,6 +31,12 @@ import (
 	"github.com/vmware/harbor/src/registryctl/client"
 )
 
+const (
+	dialConnectionTimeout = 30 * time.Second
+	dialReadTimeout       = time.Minute + 10*time.Second
+	dialWriteTimeout      = 10 * time.Second
+)
+
 // GarbageCollector is the struct to run registry's garbage collection
 type GarbageCollector struct {
 	registryCtlClient client.Client
@@ -36,6 +44,7 @@ type GarbageCollector struct {
 	uiclient          *common_http.Client
 	UIURL             string
 	insecure          bool
+	redisURL          string
 }
 
 // MaxFails implements the interface in job/Interface
@@ -55,7 +64,7 @@ func (gc *GarbageCollector) Validate(params map[string]interface{}) error {
 
 // Run implements the interface in job/Interface
 func (gc *GarbageCollector) Run(ctx env.JobContext, params map[string]interface{}) error {
-	if err := gc.init(ctx); err != nil {
+	if err := gc.init(ctx, params); err != nil {
 		return err
 	}
 	if err := gc.readonly(true); err != nil {
@@ -72,12 +81,15 @@ func (gc *GarbageCollector) Run(ctx env.JobContext, params map[string]interface{
 		gc.logger.Errorf("failed to get gc result: %v", err)
 		return err
 	}
+	if err := gc.cleanCache(); err != nil {
+		return err
+	}
 	gc.logger.Infof("GC results: status: %t, message: %s, start: %s, end: %s.", gcr.Status, gcr.Msg, gcr.StartTime, gcr.EndTime)
 	gc.logger.Infof("success to run gc in job.")
 	return nil
 }
 
-func (gc *GarbageCollector) init(ctx env.JobContext) error {
+func (gc *GarbageCollector) init(ctx env.JobContext, params map[string]interface{}) error {
 	registryctl.Init()
 	gc.registryCtlClient = registryctl.RegistryCtlClient
 	gc.logger = ctx.GetLogger()
@@ -92,6 +104,7 @@ func (gc *GarbageCollector) init(ctx env.JobContext) error {
 	} else {
 		return fmt.Errorf(errTpl, common.UIURL)
 	}
+	gc.redisURL = params["redis_url_reg"].(string)
 	return nil
 }
 
@@ -107,3 +120,30 @@ func (gc *GarbageCollector) readonly(switcher bool) error {
 	gc.logger.Info("the readonly request has been sent successfully")
 	return nil
 }
+
+// cleanCache is to clean the registry cache for GC.
+// To do this is because the issue https://github.com/docker/distribution/issues/2094
+func (gc *GarbageCollector) cleanCache() error {
+
+	con, err := redis.DialURL(
+		gc.redisURL,
+		redis.DialConnectTimeout(dialConnectionTimeout),
+		redis.DialReadTimeout(dialReadTimeout),
+		redis.DialWriteTimeout(dialWriteTimeout),
+	)
+
+	if err != nil {
+		gc.logger.Errorf("failed to connect to redis %v", err)
+		return err
+	}
+	defer con.Close()
+
+	// clean all keys in registry redis DB.
+	_, err = con.Do("FLUSHDB")
+	if err != nil {
+		gc.logger.Errorf("failed to clean registry cache %v", err)
+		return err
+	}
+
+	return nil
+}

src/ui/api/models/reg_gc.go

@@ -39,9 +39,10 @@ const (
 
 // GCReq holds request information for admin job
 type GCReq struct {
-	Schedule *ScheduleParam `json:"schedule"`
-	Status   string         `json:"status"`
-	ID       int64          `json:"id"`
+	Schedule   *ScheduleParam         `json:"schedule"`
+	Status     string                 `json:"status"`
+	ID         int64                  `json:"id"`
+	Parameters map[string]interface{} `json:"parameters"`
 }
 
 //ScheduleParam defines the parameter of schedule trigger
@@ -88,8 +89,9 @@ func (gr *GCReq) ToJob() (*models.JobData, error) {
 	}
 
 	jobData := &models.JobData{
-		Name:     job.ImageGC,
-		Metadata: metadata,
+		Name:       job.ImageGC,
+		Parameters: gr.Parameters,
+		Metadata:   metadata,
 		StatusHook: fmt.Sprintf("%s/service/notifications/jobs/adminjob/%d",
 			config.InternalUIURL(), gr.ID),
 	}

src/ui/api/reg_gc.go

@@ -17,6 +17,7 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"os"
 	"strconv"
 
 	"github.com/vmware/harbor/src/common/dao"
@@ -209,6 +210,9 @@ func (gc *GCAPI) submitJob(gr *models.GCReq) {
 		return
 	}
 	gr.ID = id
+	gr.Parameters = map[string]interface{}{
+		"redis_url_reg": os.Getenv("_REDIS_URL_REG"),
+	}
 	job, err := gr.ToJob()
 	if err != nil {
 		gc.HandleInternalServerError(fmt.Sprintf("%v", err))