Merge branch 'new-ui-with-sync-image' of https://github.com/vmware/harbor into new-ui-with-sync-image

Deploy/prepare

@@ -98,7 +98,8 @@ render(os.path.join(templates_dir, "ui", "env"),
         ldap_url=ldap_url,
         ldap_basedn=ldap_basedn,
 	self_registration=self_registration,
-        ui_secret=ui_secret)
+        ui_secret=ui_secret,
+		verify_remote_cert=verify_remote_cert)
 
 render(os.path.join(templates_dir, "ui", "app.conf"),
         ui_conf,

Deploy/templates/ui/env

@@ -17,3 +17,4 @@ LOG_LEVEL=debug
 GODEBUG=netdns=cgo
 EXT_ENDPOINT=$ui_url
 TOKEN_URL=http://ui
+VERIFY_REMOTE_CERT=$verify_remote_cert

api/base.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"os"
 	"strconv"
 
 	"github.com/astaxie/beego/validation"
@@ -136,3 +137,14 @@ func (b *BaseAPI) GetIDFromURL() int64 {
 
 	return id
 }
+
+func getIsInsecure() bool {
+	insecure := false
+
+	verifyRemoteCert := os.Getenv("VERIFY_REMOTE_CERT")
+	if verifyRemoteCert == "off" {
+		insecure = true
+	}
+
+	return insecure
+}

api/replication_job.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/vmware/harbor/dao"
 	"github.com/vmware/harbor/models"
@@ -59,6 +60,7 @@ func (ra *RepJobAPI) Prepare() {
 func (ra *RepJobAPI) List() {
 	var policyID int64
 	var repository, status string
+	var startTime, endTime *time.Time
 	var err error
 
 	policyIDStr := ra.GetString("policy_id")
@@ -69,10 +71,36 @@ func (ra *RepJobAPI) List() {
 		}
 	}
 
+	endTimeStr := ra.GetString("end_time")
+	if len(endTimeStr) != 0 {
+		i, err := strconv.ParseInt(endTimeStr, 10, 64)
+		if err != nil {
+			ra.CustomAbort(http.StatusBadRequest, "invalid end_time")
+		}
+		t := time.Unix(i, 0)
+		endTime = &t
+	}
+
+	startTimeStr := ra.GetString("start_time")
+	if len(startTimeStr) != 0 {
+		i, err := strconv.ParseInt(startTimeStr, 10, 64)
+		if err != nil {
+			ra.CustomAbort(http.StatusBadRequest, "invalid start_time")
+		}
+		t := time.Unix(i, 0)
+		startTime = &t
+	}
+
+	if startTime == nil && endTime == nil {
+		// if start_time and end_time are both null, list jobs of last 10 days
+		t := time.Now().UTC().AddDate(0, 0, -10)
+		startTime = &t
+	}
+
 	repository = ra.GetString("repository")
 	status = ra.GetString("status")
 
-	jobs, err := dao.FilterRepJobs(policyID, repository, status)
+	jobs, err := dao.FilterRepJobs(policyID, repository, status, startTime, endTime, 1000)
 	if err != nil {
 		log.Errorf("failed to filter jobs according policy ID %d, repository %s, status %s: %v", policyID, repository, status, err)
 		ra.RenderError(http.StatusInternalServerError, "Failed to query job")

api/repository.go

@@ -144,6 +144,18 @@ func (ra *RepositoryAPI) Delete() {
 		tags = append(tags, tag)
 	}
 
+	project := ""
+	if strings.Contains(repoName, "/") {
+		project = repoName[0:strings.LastIndex(repoName, "/")]
+	}
+	user, _, ok := ra.Ctx.Request.BasicAuth()
+	if !ok {
+		user, err = ra.getUsername()
+		if err != nil {
+			log.Errorf("failed to get user: %v", err)
+		}
+	}
+
 	for _, t := range tags {
 		if err := rc.DeleteTag(t); err != nil {
 			if regErr, ok := err.(*registry_error.Error); ok {
@@ -156,6 +168,11 @@ func (ra *RepositoryAPI) Delete() {
 		log.Infof("delete tag: %s %s", repoName, t)
 		go TriggerReplicationByRepository(repoName, []string{t}, models.RepOpDelete)
 
+		go func(tag string) {
+			if err := dao.AccessLog(user, project, repoName, tag, "delete"); err != nil {
+				log.Errorf("failed to add access log: %v", err)
+			}
+		}(t)
 	}
 
 	go func() {
@@ -164,7 +181,6 @@ func (ra *RepositoryAPI) Delete() {
 			log.Errorf("error occurred while refresh catalog cache: %v", err)
 		}
 	}()
-
 }
 
 type tag struct {
@@ -255,12 +271,10 @@ func (ra *RepositoryAPI) GetManifests() {
 
 func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) {
 	endpoint := os.Getenv("REGISTRY_URL")
-	// TODO read variable from config file
-	insecure := true
 
 	username, password, ok := ra.Ctx.Request.BasicAuth()
 	if ok {
-		return newRepositoryClient(endpoint, insecure, username, password,
+		return newRepositoryClient(endpoint, getIsInsecure(), username, password,
 			repoName, "repository", repoName, "pull", "push", "*")
 	}
 
@@ -269,7 +283,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo
 		return nil, err
 	}
 
-	return cache.NewRepositoryClient(endpoint, insecure, username, repoName,
+	return cache.NewRepositoryClient(endpoint, getIsInsecure(), username, repoName,
 		"repository", repoName, "pull", "push", "*")
 }
 

api/target.go

@@ -92,9 +92,7 @@ func (t *TargetAPI) Ping() {
 		password = t.GetString("password")
 	}
 
-	// TODO read variable from config file
-	insecure := true
-	registry, err := newRegistryClient(endpoint, insecure, username, password,
+	registry, err := newRegistryClient(endpoint, getIsInsecure(), username, password,
 		"", "", "")
 	if err != nil {
 		// timeout, dns resolve error, connection refused, etc.

dao/dao_test.go

@@ -1180,7 +1180,7 @@ func TestDeleteRepJob(t *testing.T) {
 }
 
 func TestFilterRepJobs(t *testing.T) {
-	jobs, err := FilterRepJobs(policyID, "", "")
+	jobs, err := FilterRepJobs(policyID, "", "", nil, nil, 1000)
 	if err != nil {
 		log.Errorf("Error occured in FilterRepJobs: %v, policy ID: %d", err, policyID)
 		return

dao/replication_job.go

@@ -311,7 +311,8 @@ func GetRepJobByPolicy(policyID int64) ([]*models.RepJob, error) {
 }
 
 // FilterRepJobs filters jobs by repo and policy ID
-func FilterRepJobs(policyID int64, repository, status string) ([]*models.RepJob, error) {
+func FilterRepJobs(policyID int64, repository, status string, startTime,
+	endTime *time.Time, limit int) ([]*models.RepJob, error) {
 	o := GetOrmer()
 
 	qs := o.QueryTable(new(models.RepJob))
@@ -324,6 +325,21 @@ func FilterRepJobs(policyID int64, repository, status string) ([]*models.RepJob,
 	if len(status) != 0 {
 		qs = qs.Filter("Status__icontains", status)
 	}
+
+	if startTime != nil {
+		fmt.Printf("%v\n", startTime)
+		qs = qs.Filter("CreationTime__gte", startTime)
+	}
+
+	if endTime != nil {
+		fmt.Printf("%v\n", endTime)
+		qs = qs.Filter("CreationTime__lte", endTime)
+	}
+
+	if limit != 0 {
+		qs = qs.Limit(limit)
+	}
+
 	qs = qs.OrderBy("-CreationTime")
 
 	var jobs []*models.RepJob

service/notification.go

@@ -39,55 +39,85 @@ const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+json
 // Post handles POST request, and records audit log or refreshes cache based on event.
 func (n *NotificationHandler) Post() {
 	var notification models.Notification
-	//log.Info("Notification Handler triggered!\n")
-	//	log.Infof("request body in string: %s", string(n.Ctx.Input.CopyBody()))
 	err := json.Unmarshal(n.Ctx.Input.CopyBody(1<<32), &notification)
 
 	if err != nil {
-		log.Errorf("error while decoding json: %v", err)
+		log.Errorf("failed to decode notification: %v", err)
 		return
 	}
-	var username, action, repo, project, repoTag string
-	var matched bool
-	for _, e := range notification.Events {
-		matched, err = regexp.MatchString(manifestPattern, e.Target.MediaType)
-		if err != nil {
-			log.Errorf("Failed to match the media type against pattern, error: %v", err)
-			matched = false
+
+	events, err := filterEvents(&notification)
+	if err != nil {
+		log.Errorf("failed to filter events: %v", err)
+		return
+	}
+
+	for _, event := range events {
+		repository := event.Target.Repository
+
+		project := ""
+		if strings.Contains(repository, "/") {
+			project = repository[0:strings.LastIndex(repository, "/")]
 		}
-		if matched && (strings.HasPrefix(e.Request.UserAgent, "docker") ||
-			strings.ToLower(strings.TrimSpace(e.Request.UserAgent)) == "harbor-registry-client") {
-			username = e.Actor.Name
-			action = e.Action
-			repo = e.Target.Repository
-			repoTag = e.Target.Tag
-			log.Debugf("repo tag is : %v ", repoTag)
 
-			if strings.Contains(repo, "/") {
-				project = repo[0:strings.LastIndex(repo, "/")]
-			}
-			if username == "" {
-				username = "anonymous"
-			}
+		tag := event.Target.Tag
+		action := event.Action
 
-			if action == "pull" && username == "job-service-user" {
-				return
-			}
+		user := event.Actor.Name
+		if len(user) == 0 {
+			user = "anonymous"
+		}
 
-			go dao.AccessLog(username, project, repo, repoTag, action)
+		go func() {
+			if err := dao.AccessLog(user, project, repository, tag, action); err != nil {
+				log.Errorf("failed to add access log: %v", err)
+			}
+		}()
+		if action == "push" {
+			go func() {
+				if err := cache.RefreshCatalogCache(); err != nil {
+					log.Errorf("failed to refresh cache: %v", err)
+				}
+			}()
+
+			operation := ""
 			if action == "push" {
-				go func() {
-					err2 := cache.RefreshCatalogCache()
-					if err2 != nil {
-						log.Errorf("Error happens when refreshing cache: %v", err2)
-					}
-				}()
-
-				go api.TriggerReplicationByRepository(repo, []string{repoTag}, models.RepOpTransfer)
+				operation = models.RepOpTransfer
 			}
+
+			go api.TriggerReplicationByRepository(repository, []string{tag}, operation)
+		}
+	}
+}
+
+func filterEvents(notification *models.Notification) ([]*models.Event, error) {
+	events := []*models.Event{}
+
+	for _, event := range notification.Events {
+		isManifest, err := regexp.MatchString(manifestPattern, event.Target.MediaType)
+		if err != nil {
+			log.Errorf("failed to match the media type against pattern: %v", err)
+			continue
+		}
+
+		if !isManifest {
+			continue
+		}
+
+		//pull and push manifest by docker-client
+		if strings.HasPrefix(event.Request.UserAgent, "docker") && (event.Action == "pull" || event.Action == "push") {
+			events = append(events, &event)
+			continue
+		}
+
+		//push manifest by docker-client or job-service
+		if strings.ToLower(strings.TrimSpace(event.Request.UserAgent)) == "harbor-registry-client" && event.Action == "push" {
+			events = append(events, &event)
+			continue
 		}
 	}
 
+	return events, nil
 }
 
 // Render returns nil as it won't render any template.