Merge pull request #620 from ywk253100/pagination

support pagination for listing repositories, jobs and access logs
This commit is contained in:
Daniel Jiang 2016-08-01 18:54:49 +08:00 committed by GitHub
commit 48d4bd677a
11 changed files with 296 additions and 151 deletions

View File

@ -31,6 +31,11 @@ import (
"github.com/astaxie/beego"
)
const (
defaultPageSize int64 = 10
maxPageSize int64 = 100
)
// BaseAPI wraps common methods for controllers to host API
type BaseAPI struct {
beego.Controller
@ -138,6 +143,60 @@ func (b *BaseAPI) GetIDFromURL() int64 {
return id
}
// set "Link" and "X-Total-Count" header for pagination request
func (b *BaseAPI) setPaginationHeader(total, page, pageSize int64) {
b.Ctx.ResponseWriter.Header().Set("X-Total-Count", strconv.FormatInt(total, 10))
link := ""
// set previous link
if page > 1 && (page-1)*pageSize <= total {
u := *(b.Ctx.Request.URL)
q := u.Query()
q.Set("page", strconv.FormatInt(page-1, 10))
u.RawQuery = q.Encode()
if len(link) != 0 {
link += ", "
}
link += fmt.Sprintf("<%s>; rel=\"prev\"", u.String())
}
// set next link
if pageSize*page < total {
u := *(b.Ctx.Request.URL)
q := u.Query()
q.Set("page", strconv.FormatInt(page+1, 10))
u.RawQuery = q.Encode()
if len(link) != 0 {
link += ", "
}
link += fmt.Sprintf("<%s>; rel=\"next\"", u.String())
}
if len(link) != 0 {
b.Ctx.ResponseWriter.Header().Set("Link", link)
}
}
func (b *BaseAPI) getPaginationParams() (page, pageSize int64) {
page, err := b.GetInt64("page", 1)
if err != nil || page <= 0 {
b.CustomAbort(http.StatusBadRequest, "invalid page")
}
pageSize, err = b.GetInt64("page_size", defaultPageSize)
if err != nil || pageSize <= 0 {
b.CustomAbort(http.StatusBadRequest, "invalid page_size")
}
if pageSize > maxPageSize {
pageSize = maxPageSize
log.Debugf("the parameter page_size %d exceeds the max %d, set it to max", pageSize, maxPageSize)
}
return page, pageSize
}
func getIsInsecure() bool {
insecure := false

View File

@ -238,25 +238,25 @@ func (p *ProjectAPI) ToggleProjectPublic() {
func (p *ProjectAPI) FilterAccessLog() {
p.userID = p.ValidateUser()
var filter models.AccessLog
p.DecodeJSONReq(&filter)
var query models.AccessLog
p.DecodeJSONReq(&query)
username := filter.Username
keywords := filter.Keywords
query.ProjectID = p.projectID
query.Username = "%" + query.Username + "%"
query.BeginTime = time.Unix(query.BeginTimestamp, 0)
query.EndTime = time.Unix(query.EndTimestamp, 0)
beginTime := time.Unix(filter.BeginTimestamp, 0)
endTime := time.Unix(filter.EndTimestamp, 0)
page, pageSize := p.getPaginationParams()
query := models.AccessLog{ProjectID: p.projectID, Username: "%" + username + "%", Keywords: keywords, BeginTime: beginTime, BeginTimestamp: filter.BeginTimestamp, EndTime: endTime, EndTimestamp: filter.EndTimestamp}
log.Infof("Query AccessLog: begin: %v, end: %v, keywords: %s", query.BeginTime, query.EndTime, query.Keywords)
accessLogList, err := dao.GetAccessLogs(query)
logs, total, err := dao.GetAccessLogs(query, pageSize, pageSize*(page-1))
if err != nil {
log.Errorf("Error occurred in GetAccessLogs, error: %v", err)
p.CustomAbort(http.StatusInternalServerError, "Internal error.")
log.Errorf("failed to get access log: %v", err)
p.CustomAbort(http.StatusInternalServerError, "")
}
p.Data["json"] = accessLogList
p.setPaginationHeader(total, page, pageSize)
p.Data["json"] = logs
p.ServeJSON()
}

View File

@ -56,43 +56,28 @@ func (ra *RepJobAPI) Prepare() {
}
// List filters jobs according to the policy and repository
// List filters jobs according to the parameters
func (ra *RepJobAPI) List() {
var policyID int64
var repository, status string
var startTime, endTime *time.Time
var num int
var err error
policyIDStr := ra.GetString("policy_id")
if len(policyIDStr) != 0 {
policyID, err = strconv.ParseInt(policyIDStr, 10, 64)
if err != nil || policyID <= 0 {
ra.CustomAbort(http.StatusBadRequest, fmt.Sprintf("invalid policy ID: %s", policyIDStr))
}
policyID, err := ra.GetInt64("policy_id")
if err != nil || policyID <= 0 {
ra.CustomAbort(http.StatusBadRequest, "invalid policy_id")
}
numStr := ra.GetString("num")
if len(numStr) != 0 {
num, err = strconv.Atoi(numStr)
if err != nil {
ra.CustomAbort(http.StatusBadRequest, fmt.Sprintf("invalid num: %s", numStr))
}
}
if num <= 0 {
num = 200
policy, err := dao.GetRepPolicy(policyID)
if err != nil {
log.Errorf("failed to get policy %d: %v", policyID, err)
ra.CustomAbort(http.StatusInternalServerError, "")
}
endTimeStr := ra.GetString("end_time")
if len(endTimeStr) != 0 {
i, err := strconv.ParseInt(endTimeStr, 10, 64)
if err != nil {
ra.CustomAbort(http.StatusBadRequest, "invalid end_time")
}
t := time.Unix(i, 0)
endTime = &t
if policy == nil {
ra.CustomAbort(http.StatusNotFound, fmt.Sprintf("policy %d not found", policyID))
}
repository := ra.GetString("repository")
status := ra.GetString("status")
var startTime *time.Time
startTimeStr := ra.GetString("start_time")
if len(startTimeStr) != 0 {
i, err := strconv.ParseInt(startTimeStr, 10, 64)
@ -103,21 +88,29 @@ func (ra *RepJobAPI) List() {
startTime = &t
}
if startTime == nil && endTime == nil {
// if start_time and end_time are both null, list jobs of last 10 days
t := time.Now().UTC().AddDate(0, 0, -10)
startTime = &t
var endTime *time.Time
endTimeStr := ra.GetString("end_time")
if len(endTimeStr) != 0 {
i, err := strconv.ParseInt(endTimeStr, 10, 64)
if err != nil {
ra.CustomAbort(http.StatusBadRequest, "invalid end_time")
}
t := time.Unix(i, 0)
endTime = &t
}
repository = ra.GetString("repository")
status = ra.GetString("status")
page, pageSize := ra.getPaginationParams()
jobs, err := dao.FilterRepJobs(policyID, repository, status, startTime, endTime, num)
jobs, total, err := dao.FilterRepJobs(policyID, repository, status,
startTime, endTime, pageSize, pageSize*(page-1))
if err != nil {
log.Errorf("failed to filter jobs according policy ID %d, repository %s, status %s: %v", policyID, repository, status, err)
ra.RenderError(http.StatusInternalServerError, "Failed to query job")
return
log.Errorf("failed to filter jobs according policy ID %d, repository %s, status %s, start time %v, end time %v: %v",
policyID, repository, status, startTime, endTime, err)
ra.CustomAbort(http.StatusInternalServerError, "")
}
ra.setPaginationHeader(total, page, pageSize)
ra.Data["json"] = jobs
ra.ServeJSON()
}

View File

@ -17,6 +17,7 @@ package api
import (
"encoding/json"
"fmt"
"net/http"
"os"
"sort"
@ -34,6 +35,7 @@ import (
registry_error "github.com/vmware/harbor/utils/registry/error"
"github.com/vmware/harbor/utils"
"github.com/vmware/harbor/utils/registry/auth"
)
@ -46,23 +48,23 @@ type RepositoryAPI struct {
// Get ...
func (ra *RepositoryAPI) Get() {
projectID, err := ra.GetInt64("project_id")
if err != nil {
log.Errorf("Failed to get project id, error: %v", err)
ra.RenderError(http.StatusBadRequest, "Invalid project id")
return
}
p, err := dao.GetProjectByID(projectID)
if err != nil {
log.Errorf("Error occurred in GetProjectById, error: %v", err)
ra.CustomAbort(http.StatusInternalServerError, "Internal error.")
}
if p == nil {
log.Warningf("Project with Id: %d does not exist", projectID)
ra.RenderError(http.StatusNotFound, "")
return
if err != nil || projectID <= 0 {
ra.CustomAbort(http.StatusBadRequest, "invalid project_id")
}
if p.Public == 0 {
page, pageSize := ra.getPaginationParams()
project, err := dao.GetProjectByID(projectID)
if err != nil {
log.Errorf("failed to get project %d: %v", projectID, err)
ra.CustomAbort(http.StatusInternalServerError, "")
}
if project == nil {
ra.CustomAbort(http.StatusNotFound, fmt.Sprintf("project %d not found", projectID))
}
if project.Public == 0 {
var userID int
if svc_utils.VerifySecret(ra.Ctx.Request) {
@ -72,37 +74,47 @@ func (ra *RepositoryAPI) Get() {
}
if !checkProjectPermission(userID, projectID) {
ra.RenderError(http.StatusForbidden, "")
return
ra.CustomAbort(http.StatusForbidden, "")
}
}
repoList, err := cache.GetRepoFromCache()
if err != nil {
log.Errorf("Failed to get repo from cache, error: %v", err)
ra.RenderError(http.StatusInternalServerError, "internal sever error")
log.Errorf("failed to get repository from cache: %v", err)
ra.CustomAbort(http.StatusInternalServerError, "")
}
projectName := p.Name
repositories := []string{}
q := ra.GetString("q")
var resp []string
if len(q) > 0 {
for _, r := range repoList {
if strings.Contains(r, "/") && strings.Contains(r[strings.LastIndex(r, "/")+1:], q) && r[0:strings.LastIndex(r, "/")] == projectName {
resp = append(resp, r)
}
for _, repo := range repoList {
pn, rest := utils.ParseRepository(repo)
if project.Name != pn {
continue
}
ra.Data["json"] = resp
} else if len(projectName) > 0 {
for _, r := range repoList {
if strings.Contains(r, "/") && r[0:strings.LastIndex(r, "/")] == projectName {
resp = append(resp, r)
}
if len(q) != 0 && !strings.Contains(rest, q) {
continue
}
ra.Data["json"] = resp
} else {
ra.Data["json"] = repoList
repositories = append(repositories, repo)
}
total := int64(len(repositories))
if (page-1)*pageSize > total {
repositories = []string{}
} else {
repositories = repositories[(page-1)*pageSize:]
}
if page*pageSize <= total {
repositories = repositories[:pageSize]
}
ra.setPaginationHeader(total, page, pageSize)
ra.Data["json"] = repositories
ra.ServeJSON()
}

View File

@ -39,67 +39,78 @@ func AddAccessLog(accessLog models.AccessLog) error {
}
//GetAccessLogs gets access logs according to different conditions
func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) {
func GetAccessLogs(query models.AccessLog, limit, offset int64) ([]models.AccessLog, int64, error) {
o := GetOrmer()
sql := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time
from access_log a left join user u on a.user_id = u.user_id
condition := ` from access_log a left join user u on a.user_id = u.user_id
where a.project_id = ? `
queryParam := make([]interface{}, 1)
queryParam = append(queryParam, accessLog.ProjectID)
queryParam = append(queryParam, query.ProjectID)
if accessLog.UserID != 0 {
sql += ` and a.user_id = ? `
queryParam = append(queryParam, accessLog.UserID)
if query.UserID != 0 {
condition += ` and a.user_id = ? `
queryParam = append(queryParam, query.UserID)
}
if accessLog.Operation != "" {
sql += ` and a.operation = ? `
queryParam = append(queryParam, accessLog.Operation)
if query.Operation != "" {
condition += ` and a.operation = ? `
queryParam = append(queryParam, query.Operation)
}
if accessLog.Username != "" {
sql += ` and u.username like ? `
queryParam = append(queryParam, accessLog.Username)
if query.Username != "" {
condition += ` and u.username like ? `
queryParam = append(queryParam, query.Username)
}
if accessLog.RepoName != "" {
sql += ` and a.repo_name = ? `
queryParam = append(queryParam, accessLog.RepoName)
if query.RepoName != "" {
condition += ` and a.repo_name = ? `
queryParam = append(queryParam, query.RepoName)
}
if accessLog.RepoTag != "" {
sql += ` and a.repo_tag = ? `
queryParam = append(queryParam, accessLog.RepoTag)
if query.RepoTag != "" {
condition += ` and a.repo_tag = ? `
queryParam = append(queryParam, query.RepoTag)
}
if accessLog.Keywords != "" {
sql += ` and a.operation in ( `
keywordList := strings.Split(accessLog.Keywords, "/")
if query.Keywords != "" {
condition += ` and a.operation in ( `
keywordList := strings.Split(query.Keywords, "/")
num := len(keywordList)
for i := 0; i < num; i++ {
if keywordList[i] != "" {
if i == num-1 {
sql += `?)`
condition += `?)`
} else {
sql += `?,`
condition += `?,`
}
queryParam = append(queryParam, keywordList[i])
}
}
}
if accessLog.BeginTimestamp > 0 {
sql += ` and a.op_time >= ? `
queryParam = append(queryParam, accessLog.BeginTime)
if query.BeginTimestamp > 0 {
condition += ` and a.op_time >= ? `
queryParam = append(queryParam, query.BeginTime)
}
if accessLog.EndTimestamp > 0 {
sql += ` and a.op_time <= ? `
queryParam = append(queryParam, accessLog.EndTime)
if query.EndTimestamp > 0 {
condition += ` and a.op_time <= ? `
queryParam = append(queryParam, query.EndTime)
}
sql += ` order by a.op_time desc `
condition += ` order by a.op_time desc `
var accessLogList []models.AccessLog
_, err := o.Raw(sql, queryParam).QueryRows(&accessLogList)
totalSQL := `select count(*) ` + condition
logs := []models.AccessLog{}
var total int64
if err := o.Raw(totalSQL, queryParam).QueryRow(&total); err != nil {
return logs, 0, err
}
condition = paginateForRawSQL(condition, limit, offset)
recordsSQL := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time ` + condition
_, err := o.Raw(recordsSQL, queryParam).QueryRows(&logs)
if err != nil {
return nil, err
return logs, 0, err
}
return accessLogList, nil
return logs, total, nil
}
// AccessLog ...

View File

@ -16,6 +16,7 @@
package dao
import (
"fmt"
"net"
"os"
@ -44,7 +45,7 @@ func GenerateRandomString() (string, error) {
//InitDB initializes the database
func InitDB() {
// orm.Debug = true
// orm.Debug = true
orm.RegisterDriver("mysql", orm.DRMySQL)
addr := os.Getenv("MYSQL_HOST")
port := os.Getenv("MYSQL_PORT")
@ -89,3 +90,7 @@ func GetOrmer() orm.Ormer {
})
return globalOrm
}
func paginateForRawSQL(sql string, limit, offset int64) string {
return fmt.Sprintf("%s limit %d offset %d", sql, limit, offset)
}

View File

@ -420,7 +420,7 @@ func TestGetAccessLog(t *testing.T) {
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
}
accessLogs, err := GetAccessLogs(queryAccessLog)
accessLogs, _, err := GetAccessLogs(queryAccessLog, 1000, 0)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
@ -448,7 +448,7 @@ func TestAddAccessLog(t *testing.T) {
if err != nil {
t.Errorf("Error occurred in AddAccessLog: %v", err)
}
accessLogList, err = GetAccessLogs(accessLog)
accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
@ -477,7 +477,7 @@ func TestAccessLog(t *testing.T) {
if err != nil {
t.Errorf("Error occurred in AccessLog: %v", err)
}
accessLogList, err = GetAccessLogs(accessLog)
accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
@ -1178,7 +1178,7 @@ func TestGetRepJobByPolicy(t *testing.T) {
}
func TestFilterRepJobs(t *testing.T) {
jobs, err := FilterRepJobs(policyID, "", "", nil, nil, 1000)
jobs, _, err := FilterRepJobs(policyID, "", "", nil, nil, 1000, 0)
if err != nil {
t.Errorf("Error occured in FilterRepJobs: %v, policy ID: %d", err, policyID)
return

View File

@ -312,12 +312,14 @@ func GetRepJobByPolicy(policyID int64) ([]*models.RepJob, error) {
return res, err
}
// FilterRepJobs filters jobs by repo and policy ID
// FilterRepJobs ...
func FilterRepJobs(policyID int64, repository, status string, startTime,
endTime *time.Time, limit int) ([]*models.RepJob, error) {
o := GetOrmer()
endTime *time.Time, limit, offset int64) ([]*models.RepJob, int64, error) {
jobs := []*models.RepJob{}
qs := GetOrmer().QueryTable(new(models.RepJob))
qs := o.QueryTable(new(models.RepJob))
if policyID != 0 {
qs = qs.Filter("PolicyID", policyID)
}
@ -327,32 +329,28 @@ func FilterRepJobs(policyID int64, repository, status string, startTime,
if len(status) != 0 {
qs = qs.Filter("Status__icontains", status)
}
if startTime != nil {
fmt.Printf("%v\n", startTime)
qs = qs.Filter("CreationTime__gte", startTime)
}
if endTime != nil {
fmt.Printf("%v\n", endTime)
qs = qs.Filter("CreationTime__lte", endTime)
}
if limit != 0 {
qs = qs.Limit(limit)
}
qs = qs.OrderBy("-UpdateTime")
var jobs []*models.RepJob
_, err := qs.All(&jobs)
total, err := qs.Count()
if err != nil {
return nil, err
return jobs, 0, err
}
_, err = qs.Limit(limit).Offset(offset).All(&jobs)
if err != nil {
return jobs, 0, err
}
genTagListForJob(jobs...)
return jobs, nil
return jobs, total, nil
}
// GetRepJobToStop get jobs that are possibly being handled by workers of a certain policy.

View File

@ -17,9 +17,10 @@
package utils
import (
"github.com/vmware/harbor/utils/log"
"net/http"
"os"
"github.com/vmware/harbor/utils/log"
)
// VerifySecret verifies the UI_SECRET cookie in a http request.
@ -27,7 +28,7 @@ func VerifySecret(r *http.Request) bool {
secret := os.Getenv("UI_SECRET")
c, err := r.Cookie("uisecret")
if err != nil {
log.Errorf("Failed to get secret cookie, error: %v", err)
log.Warningf("Failed to get secret cookie, error: %v", err)
}
return c != nil && c.Value == secret
}

View File

@ -27,10 +27,8 @@ type Repository struct {
// GetProject parses the repository and return the name of project.
func (r *Repository) GetProject() string {
if !strings.ContainsRune(r.Name, '/') {
return ""
}
return r.Name[0:strings.LastIndex(r.Name, "/")]
project, _ := ParseRepository(r.Name)
return project
}
// FormatEndpoint formats endpoint
@ -55,3 +53,17 @@ func ParseEndpoint(endpoint string) (*url.URL, error) {
}
return u, nil
}
// ParseRepository splits a repository into two parts: project and rest
func ParseRepository(repository string) (project, rest string) {
repository = strings.TrimLeft(repository, "/")
repository = strings.TrimRight(repository, "/")
if !strings.ContainsRune(repository, '/') {
rest = repository
return
}
index := strings.LastIndex(repository, "/")
project = repository[0:index]
rest = repository[index+1:]
return
}

View File

@ -1,3 +1,18 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package utils
import (
@ -7,3 +22,42 @@ import (
func TestMain(t *testing.T) {
}
func TestParseRepository(t *testing.T) {
repository := "library/ubuntu"
project, rest := ParseRepository(repository)
if project != "library" {
t.Errorf("unexpected project: %s != %s", project, "library")
}
if rest != "ubuntu" {
t.Errorf("unexpected rest: %s != %s", rest, "ubuntu")
}
repository = "library/test/ubuntu"
project, rest = ParseRepository(repository)
if project != "library/test" {
t.Errorf("unexpected project: %s != %s", project, "library/test")
}
if rest != "ubuntu" {
t.Errorf("unexpected rest: %s != %s", rest, "ubuntu")
}
repository = "ubuntu"
project, rest = ParseRepository(repository)
if project != "" {
t.Errorf("unexpected project: [%s] != [%s]", project, "")
}
if rest != "ubuntu" {
t.Errorf("unexpected rest: %s != %s", rest, "ubuntu")
}
repository = ""
project, rest = ParseRepository(repository)
if project != "" {
t.Errorf("unexpected project: [%s] != [%s]", project, "")
}
if rest != "" {
t.Errorf("unexpected rest: [%s] != [%s]", rest, "")
}
}