Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2025-01-03 14:37:44 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2461 from ywk253100/170607_log_api

Browse Source 
Support query logs according to different conditions
This commit is contained in:
Wenkai Yin 2017-06-09 11:08:08 +08:00 committed by GitHub
commit f700b1bfc6
11 changed files with 195 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
docs/swagger.yaml
View File

@ -217,8 +217,8 @@ paths:
description: Project ID does not exist.
500:
description: Unexpected internal errors.
/projects/{project_id}/logs/filter:
post:
/projects/{project_id}/logs:
get:
summary: Get access logs accompany with a relevant project.
description: |
This endpoint let user search access logs filtered by operations and date time ranges.
@ -229,11 +229,36 @@ paths:
format: int64
required: true
description: Relevant project ID
- name: access_log
in: body
schema:
$ref: '#/definitions/AccessLogFilter'
description: Search results of access logs.
- name: username
in: query
type: string
required: false
description: Username of the operator.
- name: repository
in: query
type: string
required: false
description: The name of repository
- name: tag
in: query
type: string
required: false
description: The name of tag
- name: operation
in: query
type: string
required: false
description: The operation
- name: begin_timestamp
in: query
type: string
required: false
description: The begin timestamp
- name: end_timestamp
in: query
type: string
required: false
description: The end timestamp
- name: page
in: query
type: integer
@ -861,6 +886,36 @@ paths:
description: |
This endpoint let user see the recent operation logs of the projects which he is member of
parameters:
- name: username
in: query
type: string
required: false
description: Username of the operator.
- name: repository
in: query
type: string
required: false
description: The name of repository
- name: tag
in: query
type: string
required: false
description: The name of tag
- name: operation
in: query
type: string
required: false
description: The operation
- name: begin_timestamp
in: query
type: string
required: false
description: The begin timestamp
- name: end_timestamp
in: query
type: string
required: false
description: The end timestamp
- name: page
in: query
type: integer

14
src/common/dao/accesslog.go
View File

@ -67,13 +67,19 @@ func logQueryConditions(query *models.LogQueryParam) orm.QuerySeter {
qs = qs.Filter("username__contains", query.Username)
}
if len(query.Repository) != 0 {
qs = qs.Filter("repo_name", query.Repository)
qs = qs.Filter("repo_name__contains", query.Repository)
}
if len(query.Tag) != 0 {
qs = qs.Filter("repo_tag", query.Tag)
qs = qs.Filter("repo_tag__contains", query.Tag)
}
if len(query.Operations) > 0 {
qs = qs.Filter("operation__in", query.Operations)
operations := []string{}
for _, operation := range query.Operations {
if len(operation) > 0 {
operations = append(operations, operation)
}
}
if len(operations) > 0 {
qs = qs.Filter("operation__in", operations)
}
if query.BeginTime != nil {
qs = qs.Filter("op_time__gte", query.BeginTime)

20
src/common/models/accesslog.go
View File

@ -19,19 +19,15 @@ import (
)
// AccessLog holds information about logs which are used to record the actions that user take to the resourses.
// TODO remove useless attrs
type AccessLog struct {
LogID int `orm:"pk;auto;column(log_id)" json:"log_id"`
Username string `orm:"column(username)" json:"username"`
ProjectID int64 `orm:"column(project_id)" json:"project_id"`
RepoName string `orm:"column(repo_name)" json:"repo_name"`
RepoTag string `orm:"column(repo_tag)" json:"repo_tag"`
GUID string `orm:"column(GUID)" json:"guid"`
Operation string `orm:"column(operation)" json:"operation"`
OpTime time.Time `orm:"column(op_time)" json:"op_time"`
Keywords string `orm:"-" json:"keywords"`
BeginTimestamp int64 `orm:"-" json:"begin_timestamp"`
EndTimestamp int64 `orm:"-" json:"end_timestamp"`
LogID int `orm:"pk;auto;column(log_id)" json:"log_id"`
Username string `orm:"column(username)" json:"username"`
ProjectID int64 `orm:"column(project_id)" json:"project_id"`
RepoName string `orm:"column(repo_name)" json:"repo_name"`
RepoTag string `orm:"column(repo_tag)" json:"repo_tag"`
GUID string `orm:"column(GUID)" json:"guid"`
Operation string `orm:"column(operation)" json:"operation"`
OpTime time.Time `orm:"column(op_time)" json:"op_time"`
}
// LogQueryParam is used to set query conditions when listing

11
src/common/utils/utils.go
View File

@ -19,6 +19,7 @@ import (
"fmt"
"net"
"net/url"
"strconv"
"strings"
"time"
@ -114,3 +115,13 @@ func TestTCPConn(addr string, timeout, interval int) error {
return fmt.Errorf("failed to connect to tcp:%s after %d seconds", addr, timeout)
}
}
// ParseTimeStamp parse timestamp to time
func ParseTimeStamp(timestamp string) (*time.Time, error) {
i, err := strconv.ParseInt(timestamp, 10, 64)
if err != nil {
return nil, err
}
t := time.Unix(i, 0)
return &t, nil
}

20
src/common/utils/utils_test.go
View File

@ -17,8 +17,12 @@ package utils
import (
"encoding/base64"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestParseEndpoint(t *testing.T) {
@ -191,3 +195,19 @@ func TestTestTCPConn(t *testing.T) {
t.Fatalf("failed to test tcp connection of %s: %v", addr, err)
}
}
func TestParseTimeStamp(t *testing.T) {
// invalid input
_, err := ParseTimeStamp("")
assert.NotNil(t, err)
// invalid input
_, err = ParseTimeStamp("invalid")
assert.NotNil(t, err)
// valid
now := time.Now().Unix()
result, err := ParseTimeStamp(strconv.FormatInt(now, 10))
assert.Nil(t, err)
assert.Equal(t, now, result.Unix())
}

27
src/ui/api/harborapi_test.go
View File

@ -96,7 +96,7 @@ func init() {
beego.Router("/api/users/:id([0-9]+)/password", &UserAPI{}, "put:ChangePassword")
beego.Router("/api/users/:id/sysadmin", &UserAPI{}, "put:ToggleUserAdminRole")
beego.Router("/api/projects/:id/publicity", &ProjectAPI{}, "put:ToggleProjectPublic")
beego.Router("/api/projects/:id([0-9]+)/logs/filter", &ProjectAPI{}, "post:FilterAccessLog")
beego.Router("/api/projects/:id([0-9]+)/logs", &ProjectAPI{}, "get:Logs")
beego.Router("/api/projects/:pid([0-9]+)/members/?:mid", &ProjectMemberAPI{}, "get:Get;post:Post;delete:Delete;put:Put")
beego.Router("/api/repositories", &RepositoryAPI{})
beego.Router("/api/statistics", &StatisticAPI{})
@ -379,27 +379,12 @@ func (a testapi) ToggleProjectPublicity(prjUsr usrInfo, projectID string, ispubl
}
//Get access logs accompany with a relevant project.
func (a testapi) ProjectLogsFilter(prjUsr usrInfo, projectID string, accessLog apilib.AccessLogFilter) (int, []byte, error) {
//func (a testapi) ProjectLogsFilter(prjUsr usrInfo, projectID string, accessLog apilib.AccessLog) (int, apilib.AccessLog, error) {
_sling := sling.New().Post(a.basePath)
func (a testapi) ProjectLogs(prjUsr usrInfo, projectID string, query *apilib.LogQuery) (int, []byte, error) {
_sling := sling.New().Get(a.basePath).
Path("/api/projects/" + projectID + "/logs").
QueryStruct(query)
path := "/api/projects/" + projectID + "/logs/filter"
_sling = _sling.Path(path)
// body params
_sling = _sling.BodyJSON(accessLog)
//var successPayload []apilib.AccessLog
httpStatusCode, body, err := request(_sling, jsonAcceptHeader, prjUsr)
/*
if err == nil && httpStatusCode == 200 {
err = json.Unmarshal(body, &successPayload)
}
*/
return httpStatusCode, body, err
// return httpStatusCode, successPayload, err
return request(_sling, jsonAcceptHeader, prjUsr)
}
//-------------------------Member Test---------------------------------------//

25
src/ui/api/log.go
View File

@ -19,6 +19,7 @@ import (
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils"
)
//LogAPI handles request api/logs
@ -43,12 +44,36 @@ func (l *LogAPI) Prepare() {
func (l *LogAPI) Get() {
page, size := l.GetPaginationParams()
query := &models.LogQueryParam{
Username: l.GetString("username"),
Repository: l.GetString("repository"),
Tag: l.GetString("tag"),
Operations: l.GetStrings("operation"),
Pagination: &models.Pagination{
Page: page,
Size: size,
},
}
timestamp := l.GetString("begin_timestamp")
if len(timestamp) > 0 {
t, err := utils.ParseTimeStamp(timestamp)
if err != nil {
l.HandleBadRequest(fmt.Sprintf("invalid begin_timestamp: %s", timestamp))
return
}
query.BeginTime = t
}
timestamp = l.GetString("end_timestamp")
if len(timestamp) > 0 {
t, err := utils.ParseTimeStamp(timestamp)
if err != nil {
l.HandleBadRequest(fmt.Sprintf("invalid end_timestamp: %s", timestamp))
return
}
query.EndTime = t
}
if !l.isSysAdmin {
projects, err := l.ProjectMgr.GetByMember(l.username)
if err != nil {

61
src/ui/api/project.go
View File

@ -18,11 +18,11 @@ import (
"fmt"
"net/http"
"regexp"
"strings"
"github.com/vmware/harbor/src/common"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/config"
@ -368,8 +368,8 @@ func (p *ProjectAPI) ToggleProjectPublic() {
}
}
// FilterAccessLog handles GET to /api/projects/{}/logs
func (p *ProjectAPI) FilterAccessLog() {
// Logs ...
func (p *ProjectAPI) Logs() {
if !p.SecurityCtx.IsAuthenticated() {
p.HandleUnauthorized()
return
@ -380,51 +380,54 @@ func (p *ProjectAPI) FilterAccessLog() {
return
}
var query models.AccessLog
p.DecodeJSONReq(&query)
queryParm := &models.LogQueryParam{
page, size := p.GetPaginationParams()
query := &models.LogQueryParam{
ProjectIDs: []int64{p.project.ProjectID},
Username: query.Username,
Repository: query.RepoName,
Tag: query.RepoTag,
Username: p.GetString("username"),
Repository: p.GetString("repository"),
Tag: p.GetString("tag"),
Operations: p.GetStrings("operation"),
Pagination: &models.Pagination{
Page: page,
Size: size,
},
}
if len(query.Keywords) > 0 {
queryParm.Operations = strings.Split(query.Keywords, "/")
timestamp := p.GetString("begin_timestamp")
if len(timestamp) > 0 {
t, err := utils.ParseTimeStamp(timestamp)
if err != nil {
p.HandleBadRequest(fmt.Sprintf("invalid begin_timestamp: %s", timestamp))
return
}
query.BeginTime = t
}
if query.BeginTimestamp > 0 {
beginTime := time.Unix(query.BeginTimestamp, 0)
queryParm.BeginTime = &beginTime
timestamp = p.GetString("end_timestamp")
if len(timestamp) > 0 {
t, err := utils.ParseTimeStamp(timestamp)
if err != nil {
p.HandleBadRequest(fmt.Sprintf("invalid end_timestamp: %s", timestamp))
return
}
query.EndTime = t
}
if query.EndTimestamp > 0 {
endTime := time.Unix(query.EndTimestamp, 0)
queryParm.EndTime = &endTime
}
page, pageSize := p.GetPaginationParams()
queryParm.Pagination = &models.Pagination{
Page: page,
Size: pageSize,
}
total, err := dao.GetTotalOfAccessLogs(queryParm)
total, err := dao.GetTotalOfAccessLogs(query)
if err != nil {
p.HandleInternalServerError(fmt.Sprintf(
"failed to get total of access log: %v", err))
return
}
logs, err := dao.GetAccessLogs(queryParm)
logs, err := dao.GetAccessLogs(query)
if err != nil {
p.HandleInternalServerError(fmt.Sprintf(
"failed to get access log: %v", err))
return
}
p.SetPaginationHeader(total, page, pageSize)
p.SetPaginationHeader(total, page, size)
p.Data["json"] = logs
p.ServeJSON()
}

18
src/ui/api/project_test.go
View File

@ -320,19 +320,19 @@ func TestProjectLogsFilter(t *testing.T) {
apiTest := newHarborAPI()
endTimestamp := time.Now().Unix()
startTimestamp := endTimestamp - 3600
accessLog := &apilib.AccessLogFilter{
query := &apilib.LogQuery{
Username: "admin",
Keywords: "",
BeginTimestamp: startTimestamp,
EndTimestamp: endTimestamp,
Repository: "",
Tag: "",
Operation: []string{""},
BeginTimestamp: 0,
EndTimestamp: time.Now().Unix(),
}
//-------------------case1: Response Code=200------------------------------//
fmt.Println("case 1: respose code:200")
projectID := "1"
httpStatusCode, _, err := apiTest.ProjectLogsFilter(*admin, projectID, *accessLog)
httpStatusCode, _, err := apiTest.ProjectLogs(*admin, projectID, query)
if err != nil {
t.Error("Error while search access logs")
t.Log(err)
@ -342,7 +342,7 @@ func TestProjectLogsFilter(t *testing.T) {
//-------------------case2: Response Code=401:User need to log in first.------------------------------//
fmt.Println("case 2: respose code:401:User need to log in first.")
projectID = "1"
httpStatusCode, _, err = apiTest.ProjectLogsFilter(*unknownUsr, projectID, *accessLog)
httpStatusCode, _, err = apiTest.ProjectLogs(*unknownUsr, projectID, query)
if err != nil {
t.Error("Error while search access logs")
t.Log(err)
@ -352,7 +352,7 @@ func TestProjectLogsFilter(t *testing.T) {
//-------------------case3: Response Code=404:Project does not exist.-------------------------//
fmt.Println("case 3: respose code:404:Illegal format of provided ID value.")
projectID = "11111"
httpStatusCode, _, err = apiTest.ProjectLogsFilter(*admin, projectID, *accessLog)
httpStatusCode, _, err = apiTest.ProjectLogs(*admin, projectID, query)
if err != nil {
t.Error("Error while search access logs")
t.Log(err)

2
src/ui/router.go
View File

@ -66,7 +66,7 @@ func initRouters() {
beego.Router("/api/projects/", &api.ProjectAPI{}, "get:List;post:Post;head:Head")
beego.Router("/api/projects/:id([0-9]+)", &api.ProjectAPI{})
beego.Router("/api/projects/:id([0-9]+)/publicity", &api.ProjectAPI{}, "put:ToggleProjectPublic")
beego.Router("/api/projects/:id([0-9]+)/logs/filter", &api.ProjectAPI{}, "post:FilterAccessLog")
beego.Router("/api/projects/:id([0-9]+)/logs", &api.ProjectAPI{}, "get:Logs")
beego.Router("/api/statistics", &api.StatisticAPI{})
beego.Router("/api/users/?:id", &api.UserAPI{})
beego.Router("/api/users/:id([0-9]+)/password", &api.UserAPI{}, "put:ChangePassword")

26
tests/apitests/apilib/access_log_filter.go
View File

@ -1,10 +1,10 @@
/*
/*
* Harbor API
*
* These APIs provide services for manipulating Harbor project.
*
* OpenAPI spec version: 0.3.0
*
*
* Generated by: https://github.com/swagger-api/swagger-codegen.git
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -22,17 +22,13 @@
package apilib
type AccessLogFilter struct {
// Relevant user's name that accessed this project.
Username string `json:"username,omitempty"`
// Operation name specified when project created.
Keywords string `json:"keywords,omitempty"`
// Begin timestamp for querying access logs.
BeginTimestamp int64 `json:"begin_timestamp,omitempty"`
// End timestamp for querying accessl logs.
EndTimestamp int64 `json:"end_timestamp,omitempty"`
type LogQuery struct {
Username string `json:"username"`
Repository string `json:"repository"`
Tag string `json:"tag"`
Operation []string `json:"operation"`
BeginTimestamp int64 `json:"begin_timestamp"`
EndTimestamp int64 `json:"end_timestamp"`
Page int64 `json:"page"`
PageSize int64 `json:"page_size"`
}