diff --git a/Deploy/prepare b/Deploy/prepare index d861f58e3..5a301ce61 100755 --- a/Deploy/prepare +++ b/Deploy/prepare @@ -98,7 +98,8 @@ render(os.path.join(templates_dir, "ui", "env"), ldap_url=ldap_url, ldap_basedn=ldap_basedn, self_registration=self_registration, - ui_secret=ui_secret) + ui_secret=ui_secret, + verify_remote_cert=verify_remote_cert) render(os.path.join(templates_dir, "ui", "app.conf"), ui_conf, diff --git a/Deploy/templates/ui/env b/Deploy/templates/ui/env index 5098fa1a7..a77452f41 100644 --- a/Deploy/templates/ui/env +++ b/Deploy/templates/ui/env @@ -17,3 +17,4 @@ LOG_LEVEL=debug GODEBUG=netdns=cgo EXT_ENDPOINT=$ui_url TOKEN_URL=http://ui +VERIFY_REMOTE_CERT=$verify_remote_cert diff --git a/api/base.go b/api/base.go index 7fac8e9b8..72f9da50b 100644 --- a/api/base.go +++ b/api/base.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "net/http" + "os" "strconv" "github.com/astaxie/beego/validation" @@ -136,3 +137,14 @@ func (b *BaseAPI) GetIDFromURL() int64 { return id } + +func getIsInsecure() bool { + insecure := false + + verifyRemoteCert := os.Getenv("VERIFY_REMOTE_CERT") + if verifyRemoteCert == "off" { + insecure = true + } + + return insecure +} diff --git a/api/replication_job.go b/api/replication_job.go index 380018c7c..7ced0bcac 100644 --- a/api/replication_job.go +++ b/api/replication_job.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "net/http" "strconv" + "time" "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" @@ -59,6 +60,7 @@ func (ra *RepJobAPI) Prepare() { func (ra *RepJobAPI) List() { var policyID int64 var repository, status string + var startTime, endTime *time.Time var err error policyIDStr := ra.GetString("policy_id") @@ -69,10 +71,36 @@ func (ra *RepJobAPI) List() { } } + endTimeStr := ra.GetString("end_time") + if len(endTimeStr) != 0 { + i, err := strconv.ParseInt(endTimeStr, 10, 64) + if err != nil { + ra.CustomAbort(http.StatusBadRequest, "invalid end_time") + } + t := time.Unix(i, 0) + endTime = &t + } + + startTimeStr := ra.GetString("start_time") + if len(startTimeStr) != 0 { + i, err := strconv.ParseInt(startTimeStr, 10, 64) + if err != nil { + ra.CustomAbort(http.StatusBadRequest, "invalid start_time") + } + t := time.Unix(i, 0) + startTime = &t + } + + if startTime == nil && endTime == nil { + // if start_time and end_time are both null, list jobs of last 10 days + t := time.Now().UTC().AddDate(0, 0, -10) + startTime = &t + } + repository = ra.GetString("repository") status = ra.GetString("status") - jobs, err := dao.FilterRepJobs(policyID, repository, status) + jobs, err := dao.FilterRepJobs(policyID, repository, status, startTime, endTime, 1000) if err != nil { log.Errorf("failed to filter jobs according policy ID %d, repository %s, status %s: %v", policyID, repository, status, err) ra.RenderError(http.StatusInternalServerError, "Failed to query job") diff --git a/api/repository.go b/api/repository.go index b4b36c090..e90192a2a 100644 --- a/api/repository.go +++ b/api/repository.go @@ -144,6 +144,18 @@ func (ra *RepositoryAPI) Delete() { tags = append(tags, tag) } + project := "" + if strings.Contains(repoName, "/") { + project = repoName[0:strings.LastIndex(repoName, "/")] + } + user, _, ok := ra.Ctx.Request.BasicAuth() + if !ok { + user, err = ra.getUsername() + if err != nil { + log.Errorf("failed to get user: %v", err) + } + } + for _, t := range tags { if err := rc.DeleteTag(t); err != nil { if regErr, ok := err.(*registry_error.Error); ok { @@ -156,6 +168,11 @@ func (ra *RepositoryAPI) Delete() { log.Infof("delete tag: %s %s", repoName, t) go TriggerReplicationByRepository(repoName, []string{t}, models.RepOpDelete) + go func(tag string) { + if err := dao.AccessLog(user, project, repoName, tag, "delete"); err != nil { + log.Errorf("failed to add access log: %v", err) + } + }(t) } go func() { @@ -164,7 +181,6 @@ func (ra *RepositoryAPI) Delete() { log.Errorf("error occurred while refresh catalog cache: %v", err) } }() - } type tag struct { @@ -255,12 +271,10 @@ func (ra *RepositoryAPI) GetManifests() { func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) { endpoint := os.Getenv("REGISTRY_URL") - // TODO read variable from config file - insecure := true username, password, ok := ra.Ctx.Request.BasicAuth() if ok { - return newRepositoryClient(endpoint, insecure, username, password, + return newRepositoryClient(endpoint, getIsInsecure(), username, password, repoName, "repository", repoName, "pull", "push", "*") } @@ -269,7 +283,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - return cache.NewRepositoryClient(endpoint, insecure, username, repoName, + return cache.NewRepositoryClient(endpoint, getIsInsecure(), username, repoName, "repository", repoName, "pull", "push", "*") } diff --git a/api/target.go b/api/target.go index 2271b142f..c89f1a3fd 100644 --- a/api/target.go +++ b/api/target.go @@ -92,9 +92,7 @@ func (t *TargetAPI) Ping() { password = t.GetString("password") } - // TODO read variable from config file - insecure := true - registry, err := newRegistryClient(endpoint, insecure, username, password, + registry, err := newRegistryClient(endpoint, getIsInsecure(), username, password, "", "", "") if err != nil { // timeout, dns resolve error, connection refused, etc. diff --git a/dao/dao_test.go b/dao/dao_test.go index 919882f24..5a17986fc 100644 --- a/dao/dao_test.go +++ b/dao/dao_test.go @@ -1180,7 +1180,7 @@ func TestDeleteRepJob(t *testing.T) { } func TestFilterRepJobs(t *testing.T) { - jobs, err := FilterRepJobs(policyID, "", "") + jobs, err := FilterRepJobs(policyID, "", "", nil, nil, 1000) if err != nil { log.Errorf("Error occured in FilterRepJobs: %v, policy ID: %d", err, policyID) return diff --git a/dao/replication_job.go b/dao/replication_job.go index 6de5af82d..f0d4782cd 100644 --- a/dao/replication_job.go +++ b/dao/replication_job.go @@ -311,7 +311,8 @@ func GetRepJobByPolicy(policyID int64) ([]*models.RepJob, error) { } // FilterRepJobs filters jobs by repo and policy ID -func FilterRepJobs(policyID int64, repository, status string) ([]*models.RepJob, error) { +func FilterRepJobs(policyID int64, repository, status string, startTime, + endTime *time.Time, limit int) ([]*models.RepJob, error) { o := GetOrmer() qs := o.QueryTable(new(models.RepJob)) @@ -324,6 +325,21 @@ func FilterRepJobs(policyID int64, repository, status string) ([]*models.RepJob, if len(status) != 0 { qs = qs.Filter("Status__icontains", status) } + + if startTime != nil { + fmt.Printf("%v\n", startTime) + qs = qs.Filter("CreationTime__gte", startTime) + } + + if endTime != nil { + fmt.Printf("%v\n", endTime) + qs = qs.Filter("CreationTime__lte", endTime) + } + + if limit != 0 { + qs = qs.Limit(limit) + } + qs = qs.OrderBy("-CreationTime") var jobs []*models.RepJob diff --git a/service/notification.go b/service/notification.go index 2bd391924..77ade18d3 100644 --- a/service/notification.go +++ b/service/notification.go @@ -39,55 +39,85 @@ const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+json // Post handles POST request, and records audit log or refreshes cache based on event. func (n *NotificationHandler) Post() { var notification models.Notification - //log.Info("Notification Handler triggered!\n") - // log.Infof("request body in string: %s", string(n.Ctx.Input.CopyBody())) err := json.Unmarshal(n.Ctx.Input.CopyBody(1<<32), ¬ification) if err != nil { - log.Errorf("error while decoding json: %v", err) + log.Errorf("failed to decode notification: %v", err) return } - var username, action, repo, project, repoTag string - var matched bool - for _, e := range notification.Events { - matched, err = regexp.MatchString(manifestPattern, e.Target.MediaType) - if err != nil { - log.Errorf("Failed to match the media type against pattern, error: %v", err) - matched = false + + events, err := filterEvents(¬ification) + if err != nil { + log.Errorf("failed to filter events: %v", err) + return + } + + for _, event := range events { + repository := event.Target.Repository + + project := "" + if strings.Contains(repository, "/") { + project = repository[0:strings.LastIndex(repository, "/")] } - if matched && (strings.HasPrefix(e.Request.UserAgent, "docker") || - strings.ToLower(strings.TrimSpace(e.Request.UserAgent)) == "harbor-registry-client") { - username = e.Actor.Name - action = e.Action - repo = e.Target.Repository - repoTag = e.Target.Tag - log.Debugf("repo tag is : %v ", repoTag) - if strings.Contains(repo, "/") { - project = repo[0:strings.LastIndex(repo, "/")] - } - if username == "" { - username = "anonymous" - } + tag := event.Target.Tag + action := event.Action - if action == "pull" && username == "job-service-user" { - return - } + user := event.Actor.Name + if len(user) == 0 { + user = "anonymous" + } - go dao.AccessLog(username, project, repo, repoTag, action) + go func() { + if err := dao.AccessLog(user, project, repository, tag, action); err != nil { + log.Errorf("failed to add access log: %v", err) + } + }() + if action == "push" { + go func() { + if err := cache.RefreshCatalogCache(); err != nil { + log.Errorf("failed to refresh cache: %v", err) + } + }() + + operation := "" if action == "push" { - go func() { - err2 := cache.RefreshCatalogCache() - if err2 != nil { - log.Errorf("Error happens when refreshing cache: %v", err2) - } - }() - - go api.TriggerReplicationByRepository(repo, []string{repoTag}, models.RepOpTransfer) + operation = models.RepOpTransfer } + + go api.TriggerReplicationByRepository(repository, []string{tag}, operation) + } + } +} + +func filterEvents(notification *models.Notification) ([]*models.Event, error) { + events := []*models.Event{} + + for _, event := range notification.Events { + isManifest, err := regexp.MatchString(manifestPattern, event.Target.MediaType) + if err != nil { + log.Errorf("failed to match the media type against pattern: %v", err) + continue + } + + if !isManifest { + continue + } + + //pull and push manifest by docker-client + if strings.HasPrefix(event.Request.UserAgent, "docker") && (event.Action == "pull" || event.Action == "push") { + events = append(events, &event) + continue + } + + //push manifest by docker-client or job-service + if strings.ToLower(strings.TrimSpace(event.Request.UserAgent)) == "harbor-registry-client" && event.Action == "push" { + events = append(events, &event) + continue } } + return events, nil } // Render returns nil as it won't render any template.