Add tag to accesslog

This commit is contained in:
hmwenchen 2016-04-22 00:28:59 +08:00
parent 98e63f2cb1
commit ab6bb58c36
13 changed files with 153 additions and 146 deletions

View File

@ -94,6 +94,7 @@ create table access_log (
user_id int NOT NULL,
project_id int NOT NULL,
repo_name varchar (40),
repo_tag varchar (20),
GUID varchar(64),
operation varchar(20) NOT NULL,
op_time timestamp,

View File

@ -31,6 +31,7 @@ import (
type ProjectAPI struct {
BaseAPI
userID int
username string
projectID int64
}
@ -183,6 +184,8 @@ func (p *ProjectAPI) FilterAccessLog() {
p.CustomAbort(http.StatusInternalServerError, "Internal error.")
}
p.Data["json"] = accessLogList
log.Errorf("--- accessLog first record: %v ---", accessLogList[0])
p.ServeJSON()
}

View File

@ -18,7 +18,6 @@ package api
import (
"encoding/json"
"net/http"
"os"
"strconv"
"strings"
"time"
@ -27,9 +26,6 @@ import (
"github.com/vmware/harbor/models"
svc_utils "github.com/vmware/harbor/service/utils"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry"
"github.com/vmware/harbor/utils/registry/auth"
"github.com/vmware/harbor/utils/registry/errors"
)
// RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put
@ -40,7 +36,6 @@ type RepositoryAPI struct {
BaseAPI
userID int
username string
registry *registry.Registry
}
// Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission.
@ -58,43 +53,6 @@ func (ra *RepositoryAPI) Prepare() {
} else {
ra.username = username
}
var client *http.Client
//no session, initialize a standard auth handler
if ra.userID == dao.NonExistUserID && len(ra.username) == 0 {
username, password, _ := ra.Ctx.Request.BasicAuth()
credential := auth.NewBasicAuthCredential(username, password)
client = registry.NewClientStandardAuthHandlerEmbeded(credential)
log.Debug("initializing standard auth handler")
} else {
// session works, initialize a username auth handler
username := ra.username
if len(username) == 0 {
user, err := dao.GetUser(models.User{
UserID: ra.userID,
})
if err != nil {
log.Errorf("error occurred whiling geting user for initializing a username auth handler: %v", err)
return
}
username = user.Username
}
client = registry.NewClientUsernameAuthHandlerEmbeded(username)
log.Debug("initializing username auth handler: %s", username)
}
endpoint := os.Getenv("REGISTRY_URL")
r, err := registry.New(endpoint, client)
if err != nil {
log.Fatalf("error occurred while initializing auth handler for repository API: %v", err)
}
ra.registry = r
}
// Get ...
@ -119,13 +77,11 @@ func (ra *RepositoryAPI) Get() {
ra.RenderError(http.StatusForbidden, "")
return
}
repoList, err := svc_utils.GetRepoFromCache()
if err != nil {
log.Errorf("Failed to get repo from cache, error: %v", err)
ra.RenderError(http.StatusInternalServerError, "internal sever error")
}
projectName := p.Name
q := ra.GetString("q")
var resp []string
@ -149,92 +105,26 @@ func (ra *RepositoryAPI) Get() {
ra.ServeJSON()
}
// Delete ...
func (ra *RepositoryAPI) Delete() {
repoName := ra.GetString("repo_name")
if len(repoName) == 0 {
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
}
tags := []string{}
tag := ra.GetString("tag")
if len(tag) == 0 {
tagList, err := ra.registry.ListTag(repoName)
if err != nil {
e, ok := errors.ParseError(err)
if ok {
log.Info(e)
ra.CustomAbort(e.StatusCode, e.Message)
} else {
log.Error(err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
}
tags = append(tags, tagList...)
} else {
tags = append(tags, tag)
}
for _, t := range tags {
if err := ra.registry.DeleteTag(repoName, t); err != nil {
e, ok := errors.ParseError(err)
if ok {
ra.CustomAbort(e.StatusCode, e.Message)
} else {
log.Error(err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
}
log.Infof("delete tag: %s %s", repoName, t)
}
go func() {
log.Debug("refreshing catalog cache")
if err := svc_utils.RefreshCatalogCache(); err != nil {
log.Errorf("error occurred while refresh catalog cache: %v", err)
}
}()
}
type tag struct {
Name string `json:"name"`
Tags []string `json:"tags"`
}
type histroyItem struct {
V1Compatibility string `json:"v1Compatibility"`
}
type manifest struct {
Name string `json:"name"`
Tag string `json:"tag"`
Architecture string `json:"architecture"`
SchemaVersion int `json:"schemaVersion"`
History []histroyItem `json:"history"`
}
// GetTags handles GET /api/repositories/tags
func (ra *RepositoryAPI) GetTags() {
var tags []string
repoName := ra.GetString("repo_name")
tags, err := ra.registry.ListTag(repoName)
result, err := svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repoName, "tags", "list"), ra.username)
if err != nil {
e, ok := errors.ParseError(err)
if ok {
log.Info(e)
ra.CustomAbort(e.StatusCode, e.Message)
} else {
log.Error(err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repoName, err)
ra.RenderError(http.StatusInternalServerError, "Failed to get repo tags")
} else {
t := tag{}
json.Unmarshal(result, &t)
tags = t.Tags
}
ra.Data["json"] = tags
ra.ServeJSON()
}
@ -246,20 +136,14 @@ func (ra *RepositoryAPI) GetManifests() {
item := models.RepoItem{}
_, _, payload, err := ra.registry.PullManifest(repoName, tag, registry.ManifestVersion1)
result, err := svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repoName, "manifests", tag), ra.username)
if err != nil {
e, ok := errors.ParseError(err)
if ok {
log.Info(e)
ra.CustomAbort(e.StatusCode, e.Message)
} else {
log.Error(err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repoName, tag, err)
ra.RenderError(http.StatusInternalServerError, "Internal Server Error")
return
}
mani := manifest{}
err = json.Unmarshal(payload, &mani)
mani := models.Manifest{}
err = json.Unmarshal(result, &mani)
if err != nil {
log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repoName, tag, err)
ra.RenderError(http.StatusInternalServerError, "Internal Server Error")
@ -273,6 +157,7 @@ func (ra *RepositoryAPI) GetManifests() {
ra.RenderError(http.StatusInternalServerError, "Internal Server Error")
return
}
item.CreatedStr = item.Created.Format("2006-01-02 15:04:05")
item.DurationDays = strconv.Itoa(int(time.Since(item.Created).Hours()/24)) + " days"
ra.Data["json"] = item

View File

@ -19,6 +19,7 @@ import (
"strings"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
"github.com/astaxie/beego/orm"
)
@ -27,14 +28,14 @@ import (
func AddAccessLog(accessLog models.AccessLog) error {
o := orm.NewOrm()
p, err := o.Raw(`insert into access_log
(user_id, project_id, repo_name, guid, operation, op_time)
values (?, ?, ?, ?, ?, now())`).Prepare()
(user_id, project_id, repo_name, repo_tag, guid, operation, op_time)
values (?, ?, ?, ?, ?, ?, now())`).Prepare()
if err != nil {
return err
}
defer p.Close()
_, err = p.Exec(accessLog.UserID, accessLog.ProjectID, accessLog.RepoName, accessLog.GUID, accessLog.Operation)
_, err = p.Exec(accessLog.UserID, accessLog.ProjectID, accessLog.RepoName, accessLog.RepoTag, accessLog.GUID, accessLog.Operation)
return err
}
@ -43,7 +44,7 @@ func AddAccessLog(accessLog models.AccessLog) error {
func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) {
o := orm.NewOrm()
sql := `select a.log_id, u.username, a.repo_name, a.operation, a.op_time
sql := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time
from access_log a left join user u on a.user_id = u.user_id
where a.project_id = ? `
queryParam := make([]interface{}, 1)
@ -95,13 +96,16 @@ func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) {
return accessLogList, nil
}
// AccessLog ...
func AccessLog(username, projectName, repoName, action string) error {
// AccessLog, invoked in service/notification.go
func AccessLog(username, projectName, repoName, repoTag, action string) error {
o := orm.NewOrm()
sql := "insert into access_log (user_id, project_id, repo_name, operation, op_time) " +
sql := "insert into access_log (user_id, project_id, repo_name, repo_tag, operation, op_time) " +
"select (select user_id as user_id from user where username=?), " +
"(select project_id as project_id from project where name=?), ?, ?, now() "
_, err := o.Raw(sql, username, projectName, repoName, action).Exec()
"(select project_id as project_id from project where name=?), ?, ?, ?, now() "
_, err := o.Raw(sql, username, projectName, repoName, repoTag, action).Exec()
if err != nil {
log.Errorf("error in AccessLog: %v ", err)
}
return err
}

View File

@ -60,7 +60,7 @@ func AddProject(project models.Project) error {
return err
}
accessLog := models.AccessLog{UserID: project.OwnerID, ProjectID: projectID, RepoName: project.Name + "/", GUID: "N/A", Operation: "create", OpTime: time.Now()}
accessLog := models.AccessLog{UserID: project.OwnerID, ProjectID: projectID, RepoName: project.Name + "/", RepoTag: "N/A", GUID: "N/A", Operation: "create", OpTime: time.Now()}
err = AddAccessLog(accessLog)
return err

View File

@ -25,6 +25,7 @@ type AccessLog struct {
UserID int `orm:"column(user_id)" json:"UserId"`
ProjectID int64 `orm:"column(project_id)" json:"ProjectId"`
RepoName string `orm:"column(repo_name)"`
RepoTag string `orm:"column(repo_tag)"`
GUID string `orm:"column(GUID)" json:"Guid"`
Operation string `orm:"column(operation)"`
OpTime time.Time `orm:"column(op_time)"`

View File

@ -40,6 +40,7 @@ type Target struct {
Digest string
Repository string
URL string `json:"Url"`
Tag string
}
// Actor holds information about actor.

View File

@ -29,6 +29,7 @@ type RepoItem struct {
ID string `json:"Id"`
Parent string `json:"Parent"`
Created time.Time `json:"Created"`
CreatedStr string `json:"CreatedStr"`
DurationDays string `json:"Duration Days"`
Author string `json:"Author"`
Architecture string `json:"Architecture"`
@ -42,3 +43,32 @@ type Tag struct {
Version string `json:"version"`
ImageID string `json:"image_id"`
}
type Manifest struct {
SchemaVersion int `json:"schemaVersion"`
Name string `json:"name"`
Tag string `json:"tag"`
Architecture string `json:"architecture"`
FsLayers []blobSumItem `json:"fsLayers"`
History []histroyItem `json:"history"`
}
type histroyItem struct {
V1Compatibility string `json:"v1Compatibility"`
}
type blobSumItem struct {
BlobSum string `json:"blobSum"`
}
type ManifestDigest struct {
MediaType string `json:"mediaType"`
SchemaVersion int `json:"schemaVersion"`
Layers []layerItem `json:"layers"`
}
type layerItem struct {
MediaType string `json:"mediaType"`
Size int `json:"size"`
Digest string `json:"digest"`
}

View File

@ -18,6 +18,7 @@ package service
import (
"encoding/json"
"regexp"
"sort"
"strings"
"github.com/vmware/harbor/dao"
@ -33,6 +34,11 @@ type NotificationHandler struct {
beego.Controller
}
type taglist struct {
Name string `json:"name"`
Tags []string `json:"tags"`
}
const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+json`
// Post handles POST request, and records audit log or refreshes cache based on event.
@ -46,7 +52,7 @@ func (n *NotificationHandler) Post() {
log.Errorf("error while decoding json: %v", err)
return
}
var username, action, repo, project string
var username, action, repo, project, repo_tag, tag_url string
var matched bool
for _, e := range notification.Events {
matched, err = regexp.MatchString(manifestPattern, e.Target.MediaType)
@ -58,13 +64,67 @@ func (n *NotificationHandler) Post() {
username = e.Actor.Name
action = e.Action
repo = e.Target.Repository
tag_url = e.Target.URL
result, err1 := svc_utils.RegistryAPIGet(tag_url, username)
if err1 != nil {
log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repo, tag_url, err1)
return
}
maniDig := models.ManifestDigest{}
err = json.Unmarshal(result, &maniDig)
if err != nil {
log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag_url, err)
return
}
var digestLayers []string
var tagLayers []string
for _, diglayer := range maniDig.Layers {
digestLayers = append(digestLayers, diglayer.Digest)
}
result, err = svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repo, "tags", "list"), username)
if err != nil {
log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repo, err)
} else {
t := taglist{}
json.Unmarshal(result, &t)
for _, tag := range t.Tags {
result, err = svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repo, "manifests", tag), username)
if err != nil {
log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repo, err)
continue
}
taginfo := models.Manifest{}
err = json.Unmarshal(result, &taginfo)
if err != nil {
log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag, err)
continue
}
for _, fslayer := range taginfo.FsLayers {
tagLayers = append(tagLayers, fslayer.BlobSum)
}
sort.Strings(digestLayers)
sort.Strings(tagLayers)
eq := compStringArray(digestLayers, tagLayers)
if eq {
repo_tag = tag
break
}
}
}
if strings.Contains(repo, "/") {
project = repo[0:strings.LastIndex(repo, "/")]
}
if username == "" {
username = "anonymous"
}
go dao.AccessLog(username, project, repo, action)
log.Debugf("repo tag is : %v ", repo_tag)
go dao.AccessLog(username, project, repo, repo_tag, action)
if action == "push" {
go func() {
err2 := svc_utils.RefreshCatalogCache()
@ -82,3 +142,21 @@ func (n *NotificationHandler) Post() {
func (n *NotificationHandler) Render() error {
return nil
}
func compStringArray(a, b []string) bool {
if a == nil && b == nil {
return true
}
if a == nil || b == nil {
return false
}
if len(a) != len(b) {
return false
}
for i, v := range a {
if v != b[i] {
return false
}
}
return true
}

View File

@ -59,6 +59,7 @@ repo = Repositories
user = Users
logs = Logs
repo_name = Repository Name
repo_tag = Tag
add_members = Add Members
operation = Operation
advance = Advanced Search

View File

@ -59,6 +59,7 @@ repo = 镜像仓库
user = 用户
logs = 日志
repo_name = 镜像名称
repo_tag = 镜像标签
add_members = 添加成员
operation = 操作
advance = 高级检索
@ -82,4 +83,4 @@ index_desc_2 = 2. 效率: 搭建组织内部的私有容器Registry服务
index_desc_3 = 3. 访问控制: 提供基于角色的访问控制,可集成企业目前拥有的用户管理系统(如:AD/LDAP
index_desc_4 = 4. 审计: 所有访问Registry服务的操作均被记录便于日后审计。
index_desc_5 = 5. 管理界面: 具有友好易用图形管理界面。
index_title = 企业级 Registry 服务
index_title = 企业级 Registry 服务

View File

@ -286,6 +286,7 @@ jQuery(function(){
'<tr>' +
'<td>' + e.Username + '</td>' +
'<td>' + e.RepoName + '</td>' +
'<td>' + e.RepoTag + '</td>' +
'<td>' + e.Operation + '</td>' +
'<td>' + moment(new Date(e.OpTime)).format("YYYY-MM-DD HH:mm:ss") + '</td>' +
'</tr>');

View File

@ -159,10 +159,11 @@
<table id="tblAccessLog" class="table table-hover" >
<thead>
<tr>
<th width="20%">{{i18n .Lang "username"}}</th>
<th width="40%">{{i18n .Lang "repo_name"}}</th>
<th width="20%">{{i18n .Lang "operation"}}</th>
<th width="20%">{{i18n .Lang "timestamp"}}</th>
<th width="15%">{{i18n .Lang "username"}}</th>
<th width="30%">{{i18n .Lang "repo_name"}}</th>
<th width="15%">{{i18n .Lang "repo_tag"}}</th>
<th width="15%">{{i18n .Lang "operation"}}</th>
<th width="15%">{{i18n .Lang "timestamp"}}</th>
</tr>
</thead>
<tbody>