From a2b6355fb593aef968599964d1dc3ffe15aa1ec8 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 28 Jun 2016 17:33:39 +0800 Subject: [PATCH 01/11] audit deleting repository --- api/repository.go | 10 ---- service/notification.go | 107 +++++++++++++++++++++++++++------------- 2 files changed, 72 insertions(+), 45 deletions(-) diff --git a/api/repository.go b/api/repository.go index b4b36c090..57564e61f 100644 --- a/api/repository.go +++ b/api/repository.go @@ -154,17 +154,7 @@ func (ra *RepositoryAPI) Delete() { ra.CustomAbort(http.StatusInternalServerError, "internal error") } log.Infof("delete tag: %s %s", repoName, t) - go TriggerReplicationByRepository(repoName, []string{t}, models.RepOpDelete) - } - - go func() { - log.Debug("refreshing catalog cache") - if err := cache.RefreshCatalogCache(); err != nil { - log.Errorf("error occurred while refresh catalog cache: %v", err) - } - }() - } type tag struct { diff --git a/service/notification.go b/service/notification.go index 2bd391924..e72b13419 100644 --- a/service/notification.go +++ b/service/notification.go @@ -39,55 +39,92 @@ const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+json // Post handles POST request, and records audit log or refreshes cache based on event. func (n *NotificationHandler) Post() { var notification models.Notification - //log.Info("Notification Handler triggered!\n") - // log.Infof("request body in string: %s", string(n.Ctx.Input.CopyBody())) + log.Infof("request body in string: %s", string(n.Ctx.Input.CopyBody(1<<32))) err := json.Unmarshal(n.Ctx.Input.CopyBody(1<<32), ¬ification) if err != nil { - log.Errorf("error while decoding json: %v", err) + log.Errorf("failed to decode notification: %v", err) return } - var username, action, repo, project, repoTag string - var matched bool - for _, e := range notification.Events { - matched, err = regexp.MatchString(manifestPattern, e.Target.MediaType) - if err != nil { - log.Errorf("Failed to match the media type against pattern, error: %v", err) - matched = false + + events, err := filterEvents(¬ification) + if err != nil { + log.Errorf("failed to filter events: %v", err) + return + } + + for _, event := range events { + repository := event.Target.Repository + + project := "" + if strings.Contains(repository, "/") { + project = repository[0:strings.LastIndex(repository, "/")] } - if matched && (strings.HasPrefix(e.Request.UserAgent, "docker") || - strings.ToLower(strings.TrimSpace(e.Request.UserAgent)) == "harbor-registry-client") { - username = e.Actor.Name - action = e.Action - repo = e.Target.Repository - repoTag = e.Target.Tag - log.Debugf("repo tag is : %v ", repoTag) - if strings.Contains(repo, "/") { - project = repo[0:strings.LastIndex(repo, "/")] - } - if username == "" { - username = "anonymous" - } + tag := event.Target.Tag + action := event.Action - if action == "pull" && username == "job-service-user" { - return - } + user := event.Actor.Name + if len(user) == 0 { + user = "anonymous" + } - go dao.AccessLog(username, project, repo, repoTag, action) + go dao.AccessLog(user, project, repository, tag, action) + if action == "push" || action == "delete" { + go func() { + if err := cache.RefreshCatalogCache(); err != nil { + log.Errorf("failed to refresh cache: %v", err) + } + }() + + operation := "" if action == "push" { - go func() { - err2 := cache.RefreshCatalogCache() - if err2 != nil { - log.Errorf("Error happens when refreshing cache: %v", err2) - } - }() - - go api.TriggerReplicationByRepository(repo, []string{repoTag}, models.RepOpTransfer) + operation = models.RepOpTransfer + } else { + operation = models.RepOpDelete } + + go api.TriggerReplicationByRepository(repository, []string{tag}, operation) + } + } +} + +func filterEvents(notification *models.Notification) ([]*models.Event, error) { + events := []*models.Event{} + + for _, event := range notification.Events { + + //delete + // TODO add tag field + if event.Action == "delete" { + events = append(events, &event) + continue + } + + isManifest, err := regexp.MatchString(manifestPattern, event.Target.MediaType) + if err != nil { + log.Errorf("failed to match the media type against pattern: %v", err) + continue + } + + if !isManifest { + continue + } + + //pull and push manifest by docker-client + if strings.HasPrefix(event.Request.UserAgent, "docker") && (event.Action == "pull" || event.Action == "push") { + events = append(events, &event) + continue + } + + //push manifest by docker-client or job-service + if strings.ToLower(strings.TrimSpace(event.Request.UserAgent)) == "harbor-registry-client" && event.Action == "push" { + events = append(events, &event) + continue } } + return events, nil } // Render returns nil as it won't render any template. From 10189913127eb8b2f803bfa4518ff707cd4fd108 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 28 Jun 2016 21:39:38 +0800 Subject: [PATCH 02/11] print log if error occurs --- service/notification.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/service/notification.go b/service/notification.go index e72b13419..ae3747b79 100644 --- a/service/notification.go +++ b/service/notification.go @@ -69,7 +69,11 @@ func (n *NotificationHandler) Post() { user = "anonymous" } - go dao.AccessLog(user, project, repository, tag, action) + go func() { + if err := dao.AccessLog(user, project, repository, tag, action); err != nil { + log.Errorf("failed to add access log: %v", err) + } + }() if action == "push" || action == "delete" { go func() { if err := cache.RefreshCatalogCache(); err != nil { From ac17546d38f317b8d26666bb7f4b86778fdc54f6 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 28 Jun 2016 21:47:52 +0800 Subject: [PATCH 03/11] remove useless log --- service/notification.go | 1 - 1 file changed, 1 deletion(-) diff --git a/service/notification.go b/service/notification.go index ae3747b79..cc1585bc3 100644 --- a/service/notification.go +++ b/service/notification.go @@ -39,7 +39,6 @@ const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+json // Post handles POST request, and records audit log or refreshes cache based on event. func (n *NotificationHandler) Post() { var notification models.Notification - log.Infof("request body in string: %s", string(n.Ctx.Input.CopyBody(1<<32))) err := json.Unmarshal(n.Ctx.Input.CopyBody(1<<32), ¬ification) if err != nil { From 60e15cc1fae8e4c776b4bd1135057385c5509251 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 29 Jun 2016 16:58:16 +0800 Subject: [PATCH 04/11] filter jobs by time --- api/replication_job.go | 30 +++++++++++++++++++++++++++++- dao/replication_job.go | 18 +++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/api/replication_job.go b/api/replication_job.go index 380018c7c..7ced0bcac 100644 --- a/api/replication_job.go +++ b/api/replication_job.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "net/http" "strconv" + "time" "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" @@ -59,6 +60,7 @@ func (ra *RepJobAPI) Prepare() { func (ra *RepJobAPI) List() { var policyID int64 var repository, status string + var startTime, endTime *time.Time var err error policyIDStr := ra.GetString("policy_id") @@ -69,10 +71,36 @@ func (ra *RepJobAPI) List() { } } + endTimeStr := ra.GetString("end_time") + if len(endTimeStr) != 0 { + i, err := strconv.ParseInt(endTimeStr, 10, 64) + if err != nil { + ra.CustomAbort(http.StatusBadRequest, "invalid end_time") + } + t := time.Unix(i, 0) + endTime = &t + } + + startTimeStr := ra.GetString("start_time") + if len(startTimeStr) != 0 { + i, err := strconv.ParseInt(startTimeStr, 10, 64) + if err != nil { + ra.CustomAbort(http.StatusBadRequest, "invalid start_time") + } + t := time.Unix(i, 0) + startTime = &t + } + + if startTime == nil && endTime == nil { + // if start_time and end_time are both null, list jobs of last 10 days + t := time.Now().UTC().AddDate(0, 0, -10) + startTime = &t + } + repository = ra.GetString("repository") status = ra.GetString("status") - jobs, err := dao.FilterRepJobs(policyID, repository, status) + jobs, err := dao.FilterRepJobs(policyID, repository, status, startTime, endTime, 1000) if err != nil { log.Errorf("failed to filter jobs according policy ID %d, repository %s, status %s: %v", policyID, repository, status, err) ra.RenderError(http.StatusInternalServerError, "Failed to query job") diff --git a/dao/replication_job.go b/dao/replication_job.go index 6de5af82d..f0d4782cd 100644 --- a/dao/replication_job.go +++ b/dao/replication_job.go @@ -311,7 +311,8 @@ func GetRepJobByPolicy(policyID int64) ([]*models.RepJob, error) { } // FilterRepJobs filters jobs by repo and policy ID -func FilterRepJobs(policyID int64, repository, status string) ([]*models.RepJob, error) { +func FilterRepJobs(policyID int64, repository, status string, startTime, + endTime *time.Time, limit int) ([]*models.RepJob, error) { o := GetOrmer() qs := o.QueryTable(new(models.RepJob)) @@ -324,6 +325,21 @@ func FilterRepJobs(policyID int64, repository, status string) ([]*models.RepJob, if len(status) != 0 { qs = qs.Filter("Status__icontains", status) } + + if startTime != nil { + fmt.Printf("%v\n", startTime) + qs = qs.Filter("CreationTime__gte", startTime) + } + + if endTime != nil { + fmt.Printf("%v\n", endTime) + qs = qs.Filter("CreationTime__lte", endTime) + } + + if limit != 0 { + qs = qs.Limit(limit) + } + qs = qs.OrderBy("-CreationTime") var jobs []*models.RepJob From 23b026655cba0a00ff29273835c8639d405bca2d Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 29 Jun 2016 17:10:17 +0800 Subject: [PATCH 05/11] handle repository deletion event in API --- api/repository.go | 8 ++++++++ service/notification.go | 12 +----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/api/repository.go b/api/repository.go index 57564e61f..f19d99a5e 100644 --- a/api/repository.go +++ b/api/repository.go @@ -154,7 +154,15 @@ func (ra *RepositoryAPI) Delete() { ra.CustomAbort(http.StatusInternalServerError, "internal error") } log.Infof("delete tag: %s %s", repoName, t) + go TriggerReplicationByRepository(repoName, []string{t}, models.RepOpDelete) } + + go func() { + log.Debug("refreshing catalog cache") + if err := cache.RefreshCatalogCache(); err != nil { + log.Errorf("error occurred while refresh catalog cache: %v", err) + } + }() } type tag struct { diff --git a/service/notification.go b/service/notification.go index cc1585bc3..77ade18d3 100644 --- a/service/notification.go +++ b/service/notification.go @@ -73,7 +73,7 @@ func (n *NotificationHandler) Post() { log.Errorf("failed to add access log: %v", err) } }() - if action == "push" || action == "delete" { + if action == "push" { go func() { if err := cache.RefreshCatalogCache(); err != nil { log.Errorf("failed to refresh cache: %v", err) @@ -83,8 +83,6 @@ func (n *NotificationHandler) Post() { operation := "" if action == "push" { operation = models.RepOpTransfer - } else { - operation = models.RepOpDelete } go api.TriggerReplicationByRepository(repository, []string{tag}, operation) @@ -96,14 +94,6 @@ func filterEvents(notification *models.Notification) ([]*models.Event, error) { events := []*models.Event{} for _, event := range notification.Events { - - //delete - // TODO add tag field - if event.Action == "delete" { - events = append(events, &event) - continue - } - isManifest, err := regexp.MatchString(manifestPattern, event.Target.MediaType) if err != nil { log.Errorf("failed to match the media type against pattern: %v", err) From 0c7f2142ff8eea9f2f18fb00f9e33dabd9f9ee5b Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 29 Jun 2016 17:15:49 +0800 Subject: [PATCH 06/11] pass go test --- dao/dao_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/dao_test.go b/dao/dao_test.go index e0c8a0fbb..60377bc49 100644 --- a/dao/dao_test.go +++ b/dao/dao_test.go @@ -1141,7 +1141,7 @@ func TestDeleteRepJob(t *testing.T) { } func TestFilterRepJobs(t *testing.T) { - jobs, err := FilterRepJobs(policyID, "", "") + jobs, err := FilterRepJobs(policyID, "", "", nil, nil, 1000) if err != nil { log.Errorf("Error occured in FilterRepJobs: %v, policy ID: %d", err, policyID) return From 2f9ace7eebbb9b7c5d2e9693618e06bcbe87e61c Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 29 Jun 2016 17:52:35 +0800 Subject: [PATCH 07/11] update --- api/repository.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/api/repository.go b/api/repository.go index f19d99a5e..5f3883246 100644 --- a/api/repository.go +++ b/api/repository.go @@ -144,6 +144,18 @@ func (ra *RepositoryAPI) Delete() { tags = append(tags, tag) } + project := "" + if strings.Contains(repoName, "/") { + project = repoName[0:strings.LastIndex(repoName, "/")] + } + user, _, ok := ra.Ctx.Request.BasicAuth() + if !ok { + user, err = ra.getUsername() + if err != nil { + log.Errorf("failed to get user: %v", err) + } + } + for _, t := range tags { if err := rc.DeleteTag(t); err != nil { if regErr, ok := err.(*registry_error.Error); ok { @@ -155,6 +167,13 @@ func (ra *RepositoryAPI) Delete() { } log.Infof("delete tag: %s %s", repoName, t) go TriggerReplicationByRepository(repoName, []string{t}, models.RepOpDelete) + + go func() { + + if err := dao.AccessLog(user, project, repoName, t, "delete"); err != nil { + log.Errorf("failed to add access log: %v", err) + } + }() } go func() { From f31d469758a53e0118f9f960b209eaa22d0b56b3 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 29 Jun 2016 18:09:47 +0800 Subject: [PATCH 08/11] pass insecure env to ui --- Deploy/prepare | 3 ++- Deploy/templates/ui/env | 1 + api/config.go | 35 +++++++++++++++++++++++++++++++++++ api/repository.go | 6 ++---- api/target.go | 4 +--- 5 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 api/config.go diff --git a/Deploy/prepare b/Deploy/prepare index d861f58e3..5a301ce61 100755 --- a/Deploy/prepare +++ b/Deploy/prepare @@ -98,7 +98,8 @@ render(os.path.join(templates_dir, "ui", "env"), ldap_url=ldap_url, ldap_basedn=ldap_basedn, self_registration=self_registration, - ui_secret=ui_secret) + ui_secret=ui_secret, + verify_remote_cert=verify_remote_cert) render(os.path.join(templates_dir, "ui", "app.conf"), ui_conf, diff --git a/Deploy/templates/ui/env b/Deploy/templates/ui/env index 5098fa1a7..a77452f41 100644 --- a/Deploy/templates/ui/env +++ b/Deploy/templates/ui/env @@ -17,3 +17,4 @@ LOG_LEVEL=debug GODEBUG=netdns=cgo EXT_ENDPOINT=$ui_url TOKEN_URL=http://ui +VERIFY_REMOTE_CERT=$verify_remote_cert diff --git a/api/config.go b/api/config.go new file mode 100644 index 000000000..aceb1c1b3 --- /dev/null +++ b/api/config.go @@ -0,0 +1,35 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "os" +) + +var ( + Insecure bool +) + +func init() { + verifyRemoteCert := os.Getenv("VERIFY_REMOTE_CERT") + if len(verifyRemoteCert) == 0 { + verifyRemoteCert = "on" + } + + if verifyRemoteCert == "off" { + Insecure = false + } +} diff --git a/api/repository.go b/api/repository.go index b4b36c090..254349901 100644 --- a/api/repository.go +++ b/api/repository.go @@ -255,12 +255,10 @@ func (ra *RepositoryAPI) GetManifests() { func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) { endpoint := os.Getenv("REGISTRY_URL") - // TODO read variable from config file - insecure := true username, password, ok := ra.Ctx.Request.BasicAuth() if ok { - return newRepositoryClient(endpoint, insecure, username, password, + return newRepositoryClient(endpoint, Insecure, username, password, repoName, "repository", repoName, "pull", "push", "*") } @@ -269,7 +267,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - return cache.NewRepositoryClient(endpoint, insecure, username, repoName, + return cache.NewRepositoryClient(endpoint, Insecure, username, repoName, "repository", repoName, "pull", "push", "*") } diff --git a/api/target.go b/api/target.go index 2271b142f..e22d91b62 100644 --- a/api/target.go +++ b/api/target.go @@ -92,9 +92,7 @@ func (t *TargetAPI) Ping() { password = t.GetString("password") } - // TODO read variable from config file - insecure := true - registry, err := newRegistryClient(endpoint, insecure, username, password, + registry, err := newRegistryClient(endpoint, Insecure, username, password, "", "", "") if err != nil { // timeout, dns resolve error, connection refused, etc. From d7f78502d7ef26781dd78afc94334c1f5460e710 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 29 Jun 2016 18:14:50 +0800 Subject: [PATCH 09/11] pass govet --- api/repository.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api/repository.go b/api/repository.go index 5f3883246..a303ff11e 100644 --- a/api/repository.go +++ b/api/repository.go @@ -168,12 +168,11 @@ func (ra *RepositoryAPI) Delete() { log.Infof("delete tag: %s %s", repoName, t) go TriggerReplicationByRepository(repoName, []string{t}, models.RepOpDelete) - go func() { - - if err := dao.AccessLog(user, project, repoName, t, "delete"); err != nil { + go func(tag string) { + if err := dao.AccessLog(user, project, repoName, tag, "delete"); err != nil { log.Errorf("failed to add access log: %v", err) } - }() + }(t) } go func() { From 5f75156e1dc3b7448b16d40adcac0ee4a6c34909 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 29 Jun 2016 18:20:53 +0800 Subject: [PATCH 10/11] pass golint --- api/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/config.go b/api/config.go index aceb1c1b3..3857fc408 100644 --- a/api/config.go +++ b/api/config.go @@ -20,6 +20,7 @@ import ( ) var ( + // Insecure represents whether verify cert if connecting to a https server. Insecure bool ) From 92193f34566d4e64beb0039dd7dc8ac6e5329962 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 29 Jun 2016 18:52:24 +0800 Subject: [PATCH 11/11] update --- api/base.go | 12 ++++++++++++ api/config.go | 36 ------------------------------------ api/repository.go | 4 ++-- api/target.go | 2 +- 4 files changed, 15 insertions(+), 39 deletions(-) delete mode 100644 api/config.go diff --git a/api/base.go b/api/base.go index 7fac8e9b8..72f9da50b 100644 --- a/api/base.go +++ b/api/base.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "net/http" + "os" "strconv" "github.com/astaxie/beego/validation" @@ -136,3 +137,14 @@ func (b *BaseAPI) GetIDFromURL() int64 { return id } + +func getIsInsecure() bool { + insecure := false + + verifyRemoteCert := os.Getenv("VERIFY_REMOTE_CERT") + if verifyRemoteCert == "off" { + insecure = true + } + + return insecure +} diff --git a/api/config.go b/api/config.go deleted file mode 100644 index 3857fc408..000000000 --- a/api/config.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - Copyright (c) 2016 VMware, Inc. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package api - -import ( - "os" -) - -var ( - // Insecure represents whether verify cert if connecting to a https server. - Insecure bool -) - -func init() { - verifyRemoteCert := os.Getenv("VERIFY_REMOTE_CERT") - if len(verifyRemoteCert) == 0 { - verifyRemoteCert = "on" - } - - if verifyRemoteCert == "off" { - Insecure = false - } -} diff --git a/api/repository.go b/api/repository.go index 254349901..c6f7ee9a2 100644 --- a/api/repository.go +++ b/api/repository.go @@ -258,7 +258,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo username, password, ok := ra.Ctx.Request.BasicAuth() if ok { - return newRepositoryClient(endpoint, Insecure, username, password, + return newRepositoryClient(endpoint, getIsInsecure(), username, password, repoName, "repository", repoName, "pull", "push", "*") } @@ -267,7 +267,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - return cache.NewRepositoryClient(endpoint, Insecure, username, repoName, + return cache.NewRepositoryClient(endpoint, getIsInsecure(), username, repoName, "repository", repoName, "pull", "push", "*") } diff --git a/api/target.go b/api/target.go index e22d91b62..c89f1a3fd 100644 --- a/api/target.go +++ b/api/target.go @@ -92,7 +92,7 @@ func (t *TargetAPI) Ping() { password = t.GetString("password") } - registry, err := newRegistryClient(endpoint, Insecure, username, password, + registry, err := newRegistryClient(endpoint, getIsInsecure(), username, password, "", "", "") if err != nil { // timeout, dns resolve error, connection refused, etc.