From ab6bb58c3636875d6aba25aa6260687783d79164 Mon Sep 17 00:00:00 2001 From: hmwenchen Date: Fri, 22 Apr 2016 00:28:59 +0800 Subject: [PATCH] Add tag to accesslog --- Deploy/db/registry.sql | 1 + api/project.go | 3 + api/repository.go | 143 +++-------------------------- dao/accesslog.go | 22 +++-- dao/project.go | 2 +- models/accesslog.go | 1 + models/notification.go | 1 + models/repo.go | 30 ++++++ service/notification.go | 82 ++++++++++++++++- static/i18n/locale_en-US.ini | 1 + static/i18n/locale_zh-CN.ini | 3 +- static/resources/js/item-detail.js | 1 + views/item-detail.tpl | 9 +- 13 files changed, 153 insertions(+), 146 deletions(-) diff --git a/Deploy/db/registry.sql b/Deploy/db/registry.sql index 9e4a342b3..bd0644b33 100644 --- a/Deploy/db/registry.sql +++ b/Deploy/db/registry.sql @@ -94,6 +94,7 @@ create table access_log ( user_id int NOT NULL, project_id int NOT NULL, repo_name varchar (40), + repo_tag varchar (20), GUID varchar(64), operation varchar(20) NOT NULL, op_time timestamp, diff --git a/api/project.go b/api/project.go index dbc0a4ecb..2d2aa1e47 100644 --- a/api/project.go +++ b/api/project.go @@ -31,6 +31,7 @@ import ( type ProjectAPI struct { BaseAPI userID int + username string projectID int64 } @@ -183,6 +184,8 @@ func (p *ProjectAPI) FilterAccessLog() { p.CustomAbort(http.StatusInternalServerError, "Internal error.") } p.Data["json"] = accessLogList + + log.Errorf("--- accessLog first record: %v ---", accessLogList[0]) p.ServeJSON() } diff --git a/api/repository.go b/api/repository.go index 6058b3608..252cb2fbc 100644 --- a/api/repository.go +++ b/api/repository.go @@ -18,7 +18,6 @@ package api import ( "encoding/json" "net/http" - "os" "strconv" "strings" "time" @@ -27,9 +26,6 @@ import ( "github.com/vmware/harbor/models" svc_utils "github.com/vmware/harbor/service/utils" "github.com/vmware/harbor/utils/log" - "github.com/vmware/harbor/utils/registry" - "github.com/vmware/harbor/utils/registry/auth" - "github.com/vmware/harbor/utils/registry/errors" ) // RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put @@ -40,7 +36,6 @@ type RepositoryAPI struct { BaseAPI userID int username string - registry *registry.Registry } // Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission. @@ -58,43 +53,6 @@ func (ra *RepositoryAPI) Prepare() { } else { ra.username = username } - - var client *http.Client - - //no session, initialize a standard auth handler - if ra.userID == dao.NonExistUserID && len(ra.username) == 0 { - username, password, _ := ra.Ctx.Request.BasicAuth() - - credential := auth.NewBasicAuthCredential(username, password) - client = registry.NewClientStandardAuthHandlerEmbeded(credential) - log.Debug("initializing standard auth handler") - - } else { - // session works, initialize a username auth handler - username := ra.username - if len(username) == 0 { - user, err := dao.GetUser(models.User{ - UserID: ra.userID, - }) - if err != nil { - log.Errorf("error occurred whiling geting user for initializing a username auth handler: %v", err) - return - } - - username = user.Username - } - - client = registry.NewClientUsernameAuthHandlerEmbeded(username) - log.Debug("initializing username auth handler: %s", username) - } - - endpoint := os.Getenv("REGISTRY_URL") - r, err := registry.New(endpoint, client) - if err != nil { - log.Fatalf("error occurred while initializing auth handler for repository API: %v", err) - } - - ra.registry = r } // Get ... @@ -119,13 +77,11 @@ func (ra *RepositoryAPI) Get() { ra.RenderError(http.StatusForbidden, "") return } - repoList, err := svc_utils.GetRepoFromCache() if err != nil { log.Errorf("Failed to get repo from cache, error: %v", err) ra.RenderError(http.StatusInternalServerError, "internal sever error") } - projectName := p.Name q := ra.GetString("q") var resp []string @@ -149,92 +105,26 @@ func (ra *RepositoryAPI) Get() { ra.ServeJSON() } -// Delete ... -func (ra *RepositoryAPI) Delete() { - repoName := ra.GetString("repo_name") - if len(repoName) == 0 { - ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") - } - - tags := []string{} - tag := ra.GetString("tag") - if len(tag) == 0 { - tagList, err := ra.registry.ListTag(repoName) - if err != nil { - e, ok := errors.ParseError(err) - if ok { - log.Info(e) - ra.CustomAbort(e.StatusCode, e.Message) - } else { - log.Error(err) - ra.CustomAbort(http.StatusInternalServerError, "internal error") - } - - } - tags = append(tags, tagList...) - - } else { - tags = append(tags, tag) - } - - for _, t := range tags { - if err := ra.registry.DeleteTag(repoName, t); err != nil { - e, ok := errors.ParseError(err) - if ok { - ra.CustomAbort(e.StatusCode, e.Message) - } else { - log.Error(err) - ra.CustomAbort(http.StatusInternalServerError, "internal error") - } - } - log.Infof("delete tag: %s %s", repoName, t) - } - - go func() { - log.Debug("refreshing catalog cache") - if err := svc_utils.RefreshCatalogCache(); err != nil { - log.Errorf("error occurred while refresh catalog cache: %v", err) - } - }() - -} - type tag struct { Name string `json:"name"` Tags []string `json:"tags"` } -type histroyItem struct { - V1Compatibility string `json:"v1Compatibility"` -} - -type manifest struct { - Name string `json:"name"` - Tag string `json:"tag"` - Architecture string `json:"architecture"` - SchemaVersion int `json:"schemaVersion"` - History []histroyItem `json:"history"` -} - // GetTags handles GET /api/repositories/tags func (ra *RepositoryAPI) GetTags() { var tags []string repoName := ra.GetString("repo_name") - - tags, err := ra.registry.ListTag(repoName) + result, err := svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repoName, "tags", "list"), ra.username) if err != nil { - e, ok := errors.ParseError(err) - if ok { - log.Info(e) - ra.CustomAbort(e.StatusCode, e.Message) - } else { - log.Error(err) - ra.CustomAbort(http.StatusInternalServerError, "internal error") - } + log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repoName, err) + ra.RenderError(http.StatusInternalServerError, "Failed to get repo tags") + } else { + t := tag{} + json.Unmarshal(result, &t) + tags = t.Tags } - ra.Data["json"] = tags ra.ServeJSON() } @@ -246,20 +136,14 @@ func (ra *RepositoryAPI) GetManifests() { item := models.RepoItem{} - _, _, payload, err := ra.registry.PullManifest(repoName, tag, registry.ManifestVersion1) + result, err := svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repoName, "manifests", tag), ra.username) if err != nil { - e, ok := errors.ParseError(err) - if ok { - log.Info(e) - ra.CustomAbort(e.StatusCode, e.Message) - } else { - log.Error(err) - ra.CustomAbort(http.StatusInternalServerError, "internal error") - } + log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repoName, tag, err) + ra.RenderError(http.StatusInternalServerError, "Internal Server Error") + return } - - mani := manifest{} - err = json.Unmarshal(payload, &mani) + mani := models.Manifest{} + err = json.Unmarshal(result, &mani) if err != nil { log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repoName, tag, err) ra.RenderError(http.StatusInternalServerError, "Internal Server Error") @@ -273,6 +157,7 @@ func (ra *RepositoryAPI) GetManifests() { ra.RenderError(http.StatusInternalServerError, "Internal Server Error") return } + item.CreatedStr = item.Created.Format("2006-01-02 15:04:05") item.DurationDays = strconv.Itoa(int(time.Since(item.Created).Hours()/24)) + " days" ra.Data["json"] = item diff --git a/dao/accesslog.go b/dao/accesslog.go index 721ac92cd..20fb20655 100644 --- a/dao/accesslog.go +++ b/dao/accesslog.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" "github.com/astaxie/beego/orm" ) @@ -27,14 +28,14 @@ import ( func AddAccessLog(accessLog models.AccessLog) error { o := orm.NewOrm() p, err := o.Raw(`insert into access_log - (user_id, project_id, repo_name, guid, operation, op_time) - values (?, ?, ?, ?, ?, now())`).Prepare() + (user_id, project_id, repo_name, repo_tag, guid, operation, op_time) + values (?, ?, ?, ?, ?, ?, now())`).Prepare() if err != nil { return err } defer p.Close() - _, err = p.Exec(accessLog.UserID, accessLog.ProjectID, accessLog.RepoName, accessLog.GUID, accessLog.Operation) + _, err = p.Exec(accessLog.UserID, accessLog.ProjectID, accessLog.RepoName, accessLog.RepoTag, accessLog.GUID, accessLog.Operation) return err } @@ -43,7 +44,7 @@ func AddAccessLog(accessLog models.AccessLog) error { func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) { o := orm.NewOrm() - sql := `select a.log_id, u.username, a.repo_name, a.operation, a.op_time + sql := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time from access_log a left join user u on a.user_id = u.user_id where a.project_id = ? ` queryParam := make([]interface{}, 1) @@ -95,13 +96,16 @@ func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) { return accessLogList, nil } -// AccessLog ... -func AccessLog(username, projectName, repoName, action string) error { +// AccessLog, invoked in service/notification.go +func AccessLog(username, projectName, repoName, repoTag, action string) error { o := orm.NewOrm() - sql := "insert into access_log (user_id, project_id, repo_name, operation, op_time) " + + sql := "insert into access_log (user_id, project_id, repo_name, repo_tag, operation, op_time) " + "select (select user_id as user_id from user where username=?), " + - "(select project_id as project_id from project where name=?), ?, ?, now() " - _, err := o.Raw(sql, username, projectName, repoName, action).Exec() + "(select project_id as project_id from project where name=?), ?, ?, ?, now() " + _, err := o.Raw(sql, username, projectName, repoName, repoTag, action).Exec() + if err != nil { + log.Errorf("error in AccessLog: %v ", err) + } return err } diff --git a/dao/project.go b/dao/project.go index d2a611e9f..a0e6e6b25 100644 --- a/dao/project.go +++ b/dao/project.go @@ -60,7 +60,7 @@ func AddProject(project models.Project) error { return err } - accessLog := models.AccessLog{UserID: project.OwnerID, ProjectID: projectID, RepoName: project.Name + "/", GUID: "N/A", Operation: "create", OpTime: time.Now()} + accessLog := models.AccessLog{UserID: project.OwnerID, ProjectID: projectID, RepoName: project.Name + "/", RepoTag: "N/A", GUID: "N/A", Operation: "create", OpTime: time.Now()} err = AddAccessLog(accessLog) return err diff --git a/models/accesslog.go b/models/accesslog.go index 150f3b588..3111e3a44 100644 --- a/models/accesslog.go +++ b/models/accesslog.go @@ -25,6 +25,7 @@ type AccessLog struct { UserID int `orm:"column(user_id)" json:"UserId"` ProjectID int64 `orm:"column(project_id)" json:"ProjectId"` RepoName string `orm:"column(repo_name)"` + RepoTag string `orm:"column(repo_tag)"` GUID string `orm:"column(GUID)" json:"Guid"` Operation string `orm:"column(operation)"` OpTime time.Time `orm:"column(op_time)"` diff --git a/models/notification.go b/models/notification.go index 0f608e2c6..3f30ae7e4 100644 --- a/models/notification.go +++ b/models/notification.go @@ -40,6 +40,7 @@ type Target struct { Digest string Repository string URL string `json:"Url"` + Tag string } // Actor holds information about actor. diff --git a/models/repo.go b/models/repo.go index 6b8de742b..44abd9d9b 100644 --- a/models/repo.go +++ b/models/repo.go @@ -29,6 +29,7 @@ type RepoItem struct { ID string `json:"Id"` Parent string `json:"Parent"` Created time.Time `json:"Created"` + CreatedStr string `json:"CreatedStr"` DurationDays string `json:"Duration Days"` Author string `json:"Author"` Architecture string `json:"Architecture"` @@ -42,3 +43,32 @@ type Tag struct { Version string `json:"version"` ImageID string `json:"image_id"` } + +type Manifest struct { + SchemaVersion int `json:"schemaVersion"` + Name string `json:"name"` + Tag string `json:"tag"` + Architecture string `json:"architecture"` + FsLayers []blobSumItem `json:"fsLayers"` + History []histroyItem `json:"history"` +} + +type histroyItem struct { + V1Compatibility string `json:"v1Compatibility"` +} + +type blobSumItem struct { + BlobSum string `json:"blobSum"` +} + +type ManifestDigest struct { + MediaType string `json:"mediaType"` + SchemaVersion int `json:"schemaVersion"` + Layers []layerItem `json:"layers"` +} + +type layerItem struct { + MediaType string `json:"mediaType"` + Size int `json:"size"` + Digest string `json:"digest"` +} diff --git a/service/notification.go b/service/notification.go index beab28778..04611ec21 100644 --- a/service/notification.go +++ b/service/notification.go @@ -18,6 +18,7 @@ package service import ( "encoding/json" "regexp" + "sort" "strings" "github.com/vmware/harbor/dao" @@ -33,6 +34,11 @@ type NotificationHandler struct { beego.Controller } +type taglist struct { + Name string `json:"name"` + Tags []string `json:"tags"` +} + const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+json` // Post handles POST request, and records audit log or refreshes cache based on event. @@ -46,7 +52,7 @@ func (n *NotificationHandler) Post() { log.Errorf("error while decoding json: %v", err) return } - var username, action, repo, project string + var username, action, repo, project, repo_tag, tag_url string var matched bool for _, e := range notification.Events { matched, err = regexp.MatchString(manifestPattern, e.Target.MediaType) @@ -58,13 +64,67 @@ func (n *NotificationHandler) Post() { username = e.Actor.Name action = e.Action repo = e.Target.Repository + tag_url = e.Target.URL + result, err1 := svc_utils.RegistryAPIGet(tag_url, username) + + if err1 != nil { + log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repo, tag_url, err1) + return + } + + maniDig := models.ManifestDigest{} + err = json.Unmarshal(result, &maniDig) + if err != nil { + log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag_url, err) + return + } + + var digestLayers []string + var tagLayers []string + for _, diglayer := range maniDig.Layers { + digestLayers = append(digestLayers, diglayer.Digest) + } + + result, err = svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repo, "tags", "list"), username) + if err != nil { + log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repo, err) + } else { + t := taglist{} + json.Unmarshal(result, &t) + for _, tag := range t.Tags { + result, err = svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repo, "manifests", tag), username) + if err != nil { + log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repo, err) + continue + } + taginfo := models.Manifest{} + err = json.Unmarshal(result, &taginfo) + if err != nil { + log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag, err) + continue + } + for _, fslayer := range taginfo.FsLayers { + tagLayers = append(tagLayers, fslayer.BlobSum) + } + + sort.Strings(digestLayers) + sort.Strings(tagLayers) + eq := compStringArray(digestLayers, tagLayers) + if eq { + repo_tag = tag + break + } + } + } + if strings.Contains(repo, "/") { project = repo[0:strings.LastIndex(repo, "/")] } if username == "" { username = "anonymous" } - go dao.AccessLog(username, project, repo, action) + log.Debugf("repo tag is : %v ", repo_tag) + go dao.AccessLog(username, project, repo, repo_tag, action) if action == "push" { go func() { err2 := svc_utils.RefreshCatalogCache() @@ -82,3 +142,21 @@ func (n *NotificationHandler) Post() { func (n *NotificationHandler) Render() error { return nil } + +func compStringArray(a, b []string) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} diff --git a/static/i18n/locale_en-US.ini b/static/i18n/locale_en-US.ini index 981213cf0..e7999607d 100644 --- a/static/i18n/locale_en-US.ini +++ b/static/i18n/locale_en-US.ini @@ -59,6 +59,7 @@ repo = Repositories user = Users logs = Logs repo_name = Repository Name +repo_tag = Tag add_members = Add Members operation = Operation advance = Advanced Search diff --git a/static/i18n/locale_zh-CN.ini b/static/i18n/locale_zh-CN.ini index 90ea4fc04..3f3ef904b 100644 --- a/static/i18n/locale_zh-CN.ini +++ b/static/i18n/locale_zh-CN.ini @@ -59,6 +59,7 @@ repo = 镜像仓库 user = 用户 logs = 日志 repo_name = 镜像名称 +repo_tag = 镜像标签 add_members = 添加成员 operation = 操作 advance = 高级检索 @@ -82,4 +83,4 @@ index_desc_2 = 2. 效率: 搭建组织内部的私有容器Registry服务，可 index_desc_3 = 3. 访问控制: 提供基于角色的访问控制，可集成企业目前拥有的用户管理系统（如:AD/LDAP）。 index_desc_4 = 4. 审计: 所有访问Registry服务的操作均被记录，便于日后审计。 index_desc_5 = 5. 管理界面: 具有友好易用图形管理界面。 -index_title = 企业级 Registry 服务 \ No newline at end of file +index_title = 企业级 Registry 服务 diff --git a/static/resources/js/item-detail.js b/static/resources/js/item-detail.js index 1b6ffa771..6f445f717 100644 --- a/static/resources/js/item-detail.js +++ b/static/resources/js/item-detail.js @@ -286,6 +286,7 @@ jQuery(function(){ '' + '' + e.Username + '' + '' + e.RepoName + '' + + '' + e.RepoTag + '' + '' + e.Operation + '' + '' + moment(new Date(e.OpTime)).format("YYYY-MM-DD HH:mm:ss") + '' + ''); diff --git a/views/item-detail.tpl b/views/item-detail.tpl index a1ce83e96..e9cdb2753 100644 --- a/views/item-detail.tpl +++ b/views/item-detail.tpl @@ -159,10 +159,11 @@ - - - - + + + + +
{{i18n .Lang "username"}}{{i18n .Lang "repo_name"}}{{i18n .Lang "operation"}}{{i18n .Lang "timestamp"}}{{i18n .Lang "username"}}{{i18n .Lang "repo_name"}}{{i18n .Lang "repo_tag"}}{{i18n .Lang "operation"}}{{i18n .Lang "timestamp"}}