Merge pull request #620 from ywk253100/pagination

support pagination for listing repositories, jobs and access logs

api/base.go

@@ -31,6 +31,11 @@ import (
 	"github.com/astaxie/beego"
 )
 
+const (
+	defaultPageSize int64 = 10
+	maxPageSize     int64 = 100
+)
+
 // BaseAPI wraps common methods for controllers to host API
 type BaseAPI struct {
 	beego.Controller
@@ -138,6 +143,60 @@ func (b *BaseAPI) GetIDFromURL() int64 {
 	return id
 }
 
+// set "Link" and "X-Total-Count" header for pagination request
+func (b *BaseAPI) setPaginationHeader(total, page, pageSize int64) {
+	b.Ctx.ResponseWriter.Header().Set("X-Total-Count", strconv.FormatInt(total, 10))
+
+	link := ""
+
+	// set previous link
+	if page > 1 && (page-1)*pageSize <= total {
+		u := *(b.Ctx.Request.URL)
+		q := u.Query()
+		q.Set("page", strconv.FormatInt(page-1, 10))
+		u.RawQuery = q.Encode()
+		if len(link) != 0 {
+			link += ", "
+		}
+		link += fmt.Sprintf("<%s>; rel=\"prev\"", u.String())
+	}
+
+	// set next link
+	if pageSize*page < total {
+		u := *(b.Ctx.Request.URL)
+		q := u.Query()
+		q.Set("page", strconv.FormatInt(page+1, 10))
+		u.RawQuery = q.Encode()
+		if len(link) != 0 {
+			link += ", "
+		}
+		link += fmt.Sprintf("<%s>; rel=\"next\"", u.String())
+	}
+
+	if len(link) != 0 {
+		b.Ctx.ResponseWriter.Header().Set("Link", link)
+	}
+}
+
+func (b *BaseAPI) getPaginationParams() (page, pageSize int64) {
+	page, err := b.GetInt64("page", 1)
+	if err != nil || page <= 0 {
+		b.CustomAbort(http.StatusBadRequest, "invalid page")
+	}
+
+	pageSize, err = b.GetInt64("page_size", defaultPageSize)
+	if err != nil || pageSize <= 0 {
+		b.CustomAbort(http.StatusBadRequest, "invalid page_size")
+	}
+
+	if pageSize > maxPageSize {
+		pageSize = maxPageSize
+		log.Debugf("the parameter page_size %d exceeds the max %d, set it to max", pageSize, maxPageSize)
+	}
+
+	return page, pageSize
+}
+
 func getIsInsecure() bool {
 	insecure := false
 

api/project.go

@@ -238,25 +238,25 @@ func (p *ProjectAPI) ToggleProjectPublic() {
 func (p *ProjectAPI) FilterAccessLog() {
 	p.userID = p.ValidateUser()
 
-	var filter models.AccessLog
-	p.DecodeJSONReq(&filter)
+	var query models.AccessLog
+	p.DecodeJSONReq(&query)
 
-	username := filter.Username
-	keywords := filter.Keywords
+	query.ProjectID = p.projectID
+	query.Username = "%" + query.Username + "%"
+	query.BeginTime = time.Unix(query.BeginTimestamp, 0)
+	query.EndTime = time.Unix(query.EndTimestamp, 0)
 
-	beginTime := time.Unix(filter.BeginTimestamp, 0)
-	endTime := time.Unix(filter.EndTimestamp, 0)
+	page, pageSize := p.getPaginationParams()
 
-	query := models.AccessLog{ProjectID: p.projectID, Username: "%" + username + "%", Keywords: keywords, BeginTime: beginTime, BeginTimestamp: filter.BeginTimestamp, EndTime: endTime, EndTimestamp: filter.EndTimestamp}
-
-	log.Infof("Query AccessLog: begin: %v, end: %v, keywords: %s", query.BeginTime, query.EndTime, query.Keywords)
-
-	accessLogList, err := dao.GetAccessLogs(query)
+	logs, total, err := dao.GetAccessLogs(query, pageSize, pageSize*(page-1))
 	if err != nil {
-		log.Errorf("Error occurred in GetAccessLogs, error: %v", err)
-		p.CustomAbort(http.StatusInternalServerError, "Internal error.")
+		log.Errorf("failed to get access log: %v", err)
+		p.CustomAbort(http.StatusInternalServerError, "")
 	}
-	p.Data["json"] = accessLogList
+
+	p.setPaginationHeader(total, page, pageSize)
+
+	p.Data["json"] = logs
 
 	p.ServeJSON()
 }

api/replication_job.go

@@ -56,43 +56,28 @@ func (ra *RepJobAPI) Prepare() {
 
 }
 
-// List filters jobs according to the policy and repository
+// List filters jobs according to the parameters
 func (ra *RepJobAPI) List() {
-	var policyID int64
-	var repository, status string
-	var startTime, endTime *time.Time
-	var num int
-	var err error
 
-	policyIDStr := ra.GetString("policy_id")
-	if len(policyIDStr) != 0 {
-		policyID, err = strconv.ParseInt(policyIDStr, 10, 64)
+	policyID, err := ra.GetInt64("policy_id")
 	if err != nil || policyID <= 0 {
-			ra.CustomAbort(http.StatusBadRequest, fmt.Sprintf("invalid policy ID: %s", policyIDStr))
-		}
+		ra.CustomAbort(http.StatusBadRequest, "invalid policy_id")
 	}
 
-	numStr := ra.GetString("num")
-	if len(numStr) != 0 {
-		num, err = strconv.Atoi(numStr)
+	policy, err := dao.GetRepPolicy(policyID)
 	if err != nil {
-			ra.CustomAbort(http.StatusBadRequest, fmt.Sprintf("invalid num: %s", numStr))
-		}
-	}
-	if num <= 0 {
-		num = 200
+		log.Errorf("failed to get policy %d: %v", policyID, err)
+		ra.CustomAbort(http.StatusInternalServerError, "")
 	}
 
-	endTimeStr := ra.GetString("end_time")
-	if len(endTimeStr) != 0 {
-		i, err := strconv.ParseInt(endTimeStr, 10, 64)
-		if err != nil {
-			ra.CustomAbort(http.StatusBadRequest, "invalid end_time")
-		}
-		t := time.Unix(i, 0)
-		endTime = &t
+	if policy == nil {
+		ra.CustomAbort(http.StatusNotFound, fmt.Sprintf("policy %d not found", policyID))
 	}
 
+	repository := ra.GetString("repository")
+	status := ra.GetString("status")
+
+	var startTime *time.Time
 	startTimeStr := ra.GetString("start_time")
 	if len(startTimeStr) != 0 {
 		i, err := strconv.ParseInt(startTimeStr, 10, 64)
@@ -103,21 +88,29 @@ func (ra *RepJobAPI) List() {
 		startTime = &t
 	}
 
-	if startTime == nil && endTime == nil {
-		// if start_time and end_time are both null, list jobs of last 10 days
-		t := time.Now().UTC().AddDate(0, 0, -10)
-		startTime = &t
-	}
-
-	repository = ra.GetString("repository")
-	status = ra.GetString("status")
-
-	jobs, err := dao.FilterRepJobs(policyID, repository, status, startTime, endTime, num)
+	var endTime *time.Time
+	endTimeStr := ra.GetString("end_time")
+	if len(endTimeStr) != 0 {
+		i, err := strconv.ParseInt(endTimeStr, 10, 64)
 		if err != nil {
-		log.Errorf("failed to filter jobs according policy ID %d, repository %s, status %s: %v", policyID, repository, status, err)
-		ra.RenderError(http.StatusInternalServerError, "Failed to query job")
-		return
+			ra.CustomAbort(http.StatusBadRequest, "invalid end_time")
 		}
+		t := time.Unix(i, 0)
+		endTime = &t
+	}
+
+	page, pageSize := ra.getPaginationParams()
+
+	jobs, total, err := dao.FilterRepJobs(policyID, repository, status,
+		startTime, endTime, pageSize, pageSize*(page-1))
+	if err != nil {
+		log.Errorf("failed to filter jobs according policy ID %d, repository %s, status %s, start time %v, end time %v: %v",
+			policyID, repository, status, startTime, endTime, err)
+		ra.CustomAbort(http.StatusInternalServerError, "")
+	}
+
+	ra.setPaginationHeader(total, page, pageSize)
+
 	ra.Data["json"] = jobs
 	ra.ServeJSON()
 }

api/repository.go

@@ -17,6 +17,7 @@ package api
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"os"
 	"sort"
@@ -34,6 +35,7 @@ import (
 
 	registry_error "github.com/vmware/harbor/utils/registry/error"
 
+	"github.com/vmware/harbor/utils"
 	"github.com/vmware/harbor/utils/registry/auth"
 )
 
@@ -46,23 +48,23 @@ type RepositoryAPI struct {
 // Get ...
 func (ra *RepositoryAPI) Get() {
 	projectID, err := ra.GetInt64("project_id")
-	if err != nil {
-		log.Errorf("Failed to get project id, error: %v", err)
-		ra.RenderError(http.StatusBadRequest, "Invalid project id")
-		return
-	}
-	p, err := dao.GetProjectByID(projectID)
-	if err != nil {
-		log.Errorf("Error occurred in GetProjectById, error: %v", err)
-		ra.CustomAbort(http.StatusInternalServerError, "Internal error.")
-	}
-	if p == nil {
-		log.Warningf("Project with Id: %d does not exist", projectID)
-		ra.RenderError(http.StatusNotFound, "")
-		return
+	if err != nil || projectID <= 0 {
+		ra.CustomAbort(http.StatusBadRequest, "invalid project_id")
 	}
 
-	if p.Public == 0 {
+	page, pageSize := ra.getPaginationParams()
+
+	project, err := dao.GetProjectByID(projectID)
+	if err != nil {
+		log.Errorf("failed to get project %d: %v", projectID, err)
+		ra.CustomAbort(http.StatusInternalServerError, "")
+	}
+
+	if project == nil {
+		ra.CustomAbort(http.StatusNotFound, fmt.Sprintf("project %d not found", projectID))
+	}
+
+	if project.Public == 0 {
 		var userID int
 
 		if svc_utils.VerifySecret(ra.Ctx.Request) {
@@ -72,37 +74,47 @@ func (ra *RepositoryAPI) Get() {
 		}
 
 		if !checkProjectPermission(userID, projectID) {
-			ra.RenderError(http.StatusForbidden, "")
-			return
+			ra.CustomAbort(http.StatusForbidden, "")
 		}
 	}
 
 	repoList, err := cache.GetRepoFromCache()
 	if err != nil {
-		log.Errorf("Failed to get repo from cache, error: %v", err)
-		ra.RenderError(http.StatusInternalServerError, "internal sever error")
+		log.Errorf("failed to get repository from cache: %v", err)
+		ra.CustomAbort(http.StatusInternalServerError, "")
 	}
 
-	projectName := p.Name
+	repositories := []string{}
+
 	q := ra.GetString("q")
-	var resp []string
-	if len(q) > 0 {
-		for _, r := range repoList {
-			if strings.Contains(r, "/") && strings.Contains(r[strings.LastIndex(r, "/")+1:], q) && r[0:strings.LastIndex(r, "/")] == projectName {
-				resp = append(resp, r)
+	for _, repo := range repoList {
+		pn, rest := utils.ParseRepository(repo)
+		if project.Name != pn {
+			continue
 		}
+
+		if len(q) != 0 && !strings.Contains(rest, q) {
+			continue
 		}
-		ra.Data["json"] = resp
-	} else if len(projectName) > 0 {
-		for _, r := range repoList {
-			if strings.Contains(r, "/") && r[0:strings.LastIndex(r, "/")] == projectName {
-				resp = append(resp, r)
+
+		repositories = append(repositories, repo)
 	}
-		}
-		ra.Data["json"] = resp
+
+	total := int64(len(repositories))
+
+	if (page-1)*pageSize > total {
+		repositories = []string{}
 	} else {
-		ra.Data["json"] = repoList
+		repositories = repositories[(page-1)*pageSize:]
 	}
+
+	if page*pageSize <= total {
+		repositories = repositories[:pageSize]
+	}
+
+	ra.setPaginationHeader(total, page, pageSize)
+
+	ra.Data["json"] = repositories
 	ra.ServeJSON()
 }
 

dao/accesslog.go

@@ -39,67 +39,78 @@ func AddAccessLog(accessLog models.AccessLog) error {
 }
 
 //GetAccessLogs gets access logs according to different conditions
-func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) {
-
+func GetAccessLogs(query models.AccessLog, limit, offset int64) ([]models.AccessLog, int64, error) {
 	o := GetOrmer()
-	sql := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time
-		from access_log a left join user u on a.user_id = u.user_id
+
+	condition := ` from access_log a left join user u on a.user_id = u.user_id
 		where a.project_id = ? `
 	queryParam := make([]interface{}, 1)
-	queryParam = append(queryParam, accessLog.ProjectID)
+	queryParam = append(queryParam, query.ProjectID)
 
-	if accessLog.UserID != 0 {
-		sql += ` and a.user_id = ? `
-		queryParam = append(queryParam, accessLog.UserID)
+	if query.UserID != 0 {
+		condition += ` and a.user_id = ? `
+		queryParam = append(queryParam, query.UserID)
 	}
-	if accessLog.Operation != "" {
-		sql += ` and a.operation = ? `
-		queryParam = append(queryParam, accessLog.Operation)
+	if query.Operation != "" {
+		condition += ` and a.operation = ? `
+		queryParam = append(queryParam, query.Operation)
 	}
-	if accessLog.Username != "" {
-		sql += ` and u.username like ? `
-		queryParam = append(queryParam, accessLog.Username)
+	if query.Username != "" {
+		condition += ` and u.username like ? `
+		queryParam = append(queryParam, query.Username)
 	}
-	if accessLog.RepoName != "" {
-		sql += ` and a.repo_name = ? `
-		queryParam = append(queryParam, accessLog.RepoName)
+	if query.RepoName != "" {
+		condition += ` and a.repo_name = ? `
+		queryParam = append(queryParam, query.RepoName)
 	}
-	if accessLog.RepoTag != "" {
-		sql += ` and a.repo_tag = ? `
-		queryParam = append(queryParam, accessLog.RepoTag)
+	if query.RepoTag != "" {
+		condition += ` and a.repo_tag = ? `
+		queryParam = append(queryParam, query.RepoTag)
 	}
-	if accessLog.Keywords != "" {
-		sql += ` and a.operation in ( `
-		keywordList := strings.Split(accessLog.Keywords, "/")
+	if query.Keywords != "" {
+		condition += ` and a.operation in ( `
+		keywordList := strings.Split(query.Keywords, "/")
 		num := len(keywordList)
 		for i := 0; i < num; i++ {
 			if keywordList[i] != "" {
 				if i == num-1 {
-					sql += `?)`
+					condition += `?)`
 				} else {
-					sql += `?,`
+					condition += `?,`
 				}
 				queryParam = append(queryParam, keywordList[i])
 			}
 		}
 	}
-	if accessLog.BeginTimestamp > 0 {
-		sql += ` and a.op_time >= ? `
-		queryParam = append(queryParam, accessLog.BeginTime)
+	if query.BeginTimestamp > 0 {
+		condition += ` and a.op_time >= ? `
+		queryParam = append(queryParam, query.BeginTime)
 	}
-	if accessLog.EndTimestamp > 0 {
-		sql += ` and a.op_time <= ? `
-		queryParam = append(queryParam, accessLog.EndTime)
+	if query.EndTimestamp > 0 {
+		condition += ` and a.op_time <= ? `
+		queryParam = append(queryParam, query.EndTime)
 	}
 
-	sql += ` order by a.op_time desc `
+	condition += ` order by a.op_time desc `
 
-	var accessLogList []models.AccessLog
-	_, err := o.Raw(sql, queryParam).QueryRows(&accessLogList)
+	totalSQL := `select count(*) ` + condition
+
+	logs := []models.AccessLog{}
+
+	var total int64
+	if err := o.Raw(totalSQL, queryParam).QueryRow(&total); err != nil {
+		return logs, 0, err
+	}
+
+	condition = paginateForRawSQL(condition, limit, offset)
+
+	recordsSQL := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time ` + condition
+	_, err := o.Raw(recordsSQL, queryParam).QueryRows(&logs)
 	if err != nil {
-		return nil, err
+		return logs, 0, err
 	}
-	return accessLogList, nil
+
+	return logs, total, nil
 }
 
 // AccessLog ...

dao/base.go

@@ -16,6 +16,7 @@
 package dao
 
 import (
+	"fmt"
 	"net"
 
 	"os"
@@ -89,3 +90,7 @@ func GetOrmer() orm.Ormer {
 	})
 	return globalOrm
 }
+
+func paginateForRawSQL(sql string, limit, offset int64) string {
+	return fmt.Sprintf("%s limit %d offset %d", sql, limit, offset)
+}

dao/dao_test.go

@@ -420,7 +420,7 @@ func TestGetAccessLog(t *testing.T) {
 		UserID:    currentUser.UserID,
 		ProjectID: currentProject.ProjectID,
 	}
-	accessLogs, err := GetAccessLogs(queryAccessLog)
+	accessLogs, _, err := GetAccessLogs(queryAccessLog, 1000, 0)
 	if err != nil {
 		t.Errorf("Error occurred in GetAccessLog: %v", err)
 	}
@@ -448,7 +448,7 @@ func TestAddAccessLog(t *testing.T) {
 	if err != nil {
 		t.Errorf("Error occurred in AddAccessLog: %v", err)
 	}
-	accessLogList, err = GetAccessLogs(accessLog)
+	accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0)
 	if err != nil {
 		t.Errorf("Error occurred in GetAccessLog: %v", err)
 	}
@@ -477,7 +477,7 @@ func TestAccessLog(t *testing.T) {
 	if err != nil {
 		t.Errorf("Error occurred in AccessLog: %v", err)
 	}
-	accessLogList, err = GetAccessLogs(accessLog)
+	accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0)
 	if err != nil {
 		t.Errorf("Error occurred in GetAccessLog: %v", err)
 	}
@@ -1178,7 +1178,7 @@ func TestGetRepJobByPolicy(t *testing.T) {
 }
 
 func TestFilterRepJobs(t *testing.T) {
-	jobs, err := FilterRepJobs(policyID, "", "", nil, nil, 1000)
+	jobs, _, err := FilterRepJobs(policyID, "", "", nil, nil, 1000, 0)
 	if err != nil {
 		t.Errorf("Error occured in FilterRepJobs: %v, policy ID: %d", err, policyID)
 		return

dao/replication_job.go

@@ -312,12 +312,14 @@ func GetRepJobByPolicy(policyID int64) ([]*models.RepJob, error) {
 	return res, err
 }
 
-// FilterRepJobs filters jobs by repo and policy ID
+// FilterRepJobs ...
 func FilterRepJobs(policyID int64, repository, status string, startTime,
-	endTime *time.Time, limit int) ([]*models.RepJob, error) {
-	o := GetOrmer()
+	endTime *time.Time, limit, offset int64) ([]*models.RepJob, int64, error) {
+
+	jobs := []*models.RepJob{}
+
+	qs := GetOrmer().QueryTable(new(models.RepJob))
 
-	qs := o.QueryTable(new(models.RepJob))
 	if policyID != 0 {
 		qs = qs.Filter("PolicyID", policyID)
 	}
@@ -327,32 +329,28 @@ func FilterRepJobs(policyID int64, repository, status string, startTime,
 	if len(status) != 0 {
 		qs = qs.Filter("Status__icontains", status)
 	}
-
 	if startTime != nil {
-		fmt.Printf("%v\n", startTime)
 		qs = qs.Filter("CreationTime__gte", startTime)
 	}
-
 	if endTime != nil {
-		fmt.Printf("%v\n", endTime)
 		qs = qs.Filter("CreationTime__lte", endTime)
 	}
 
-	if limit != 0 {
-		qs = qs.Limit(limit)
-	}
-
 	qs = qs.OrderBy("-UpdateTime")
 
-	var jobs []*models.RepJob
-	_, err := qs.All(&jobs)
+	total, err := qs.Count()
 	if err != nil {
-		return nil, err
+		return jobs, 0, err
+	}
+
+	_, err = qs.Limit(limit).Offset(offset).All(&jobs)
+	if err != nil {
+		return jobs, 0, err
 	}
 
 	genTagListForJob(jobs...)
 
-	return jobs, nil
+	return jobs, total, nil
 }
 
 // GetRepJobToStop get jobs that are possibly being handled by workers of a certain policy.

service/utils/utils.go

@@ -17,9 +17,10 @@
 package utils
 
 import (
-	"github.com/vmware/harbor/utils/log"
 	"net/http"
 	"os"
+
+	"github.com/vmware/harbor/utils/log"
 )
 
 // VerifySecret verifies the UI_SECRET cookie in a http request.
@@ -27,7 +28,7 @@ func VerifySecret(r *http.Request) bool {
 	secret := os.Getenv("UI_SECRET")
 	c, err := r.Cookie("uisecret")
 	if err != nil {
-		log.Errorf("Failed to get secret cookie, error: %v", err)
+		log.Warningf("Failed to get secret cookie, error: %v", err)
 	}
 	return c != nil && c.Value == secret
 }

utils/utils.go

@@ -27,10 +27,8 @@ type Repository struct {
 
 // GetProject parses the repository and return the name of project.
 func (r *Repository) GetProject() string {
-	if !strings.ContainsRune(r.Name, '/') {
-		return ""
-	}
-	return r.Name[0:strings.LastIndex(r.Name, "/")]
+	project, _ := ParseRepository(r.Name)
+	return project
 }
 
 // FormatEndpoint formats endpoint
@@ -55,3 +53,17 @@ func ParseEndpoint(endpoint string) (*url.URL, error) {
 	}
 	return u, nil
 }
+
+// ParseRepository splits a repository into two parts: project and rest
+func ParseRepository(repository string) (project, rest string) {
+	repository = strings.TrimLeft(repository, "/")
+	repository = strings.TrimRight(repository, "/")
+	if !strings.ContainsRune(repository, '/') {
+		rest = repository
+		return
+	}
+	index := strings.LastIndex(repository, "/")
+	project = repository[0:index]
+	rest = repository[index+1:]
+	return
+}

utils/utils_test.go

@@ -1,3 +1,18 @@
+/*
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package utils
 
 import (
@@ -7,3 +22,42 @@ import (
 func TestMain(t *testing.T) {
 }
 
+func TestParseRepository(t *testing.T) {
+	repository := "library/ubuntu"
+	project, rest := ParseRepository(repository)
+	if project != "library" {
+		t.Errorf("unexpected project: %s != %s", project, "library")
+	}
+	if rest != "ubuntu" {
+		t.Errorf("unexpected rest: %s != %s", rest, "ubuntu")
+	}
+
+	repository = "library/test/ubuntu"
+	project, rest = ParseRepository(repository)
+	if project != "library/test" {
+		t.Errorf("unexpected project: %s != %s", project, "library/test")
+	}
+	if rest != "ubuntu" {
+		t.Errorf("unexpected rest: %s != %s", rest, "ubuntu")
+	}
+
+	repository = "ubuntu"
+	project, rest = ParseRepository(repository)
+	if project != "" {
+		t.Errorf("unexpected project: [%s] != [%s]", project, "")
+	}
+
+	if rest != "ubuntu" {
+		t.Errorf("unexpected rest: %s != %s", rest, "ubuntu")
+	}
+
+	repository = ""
+	project, rest = ParseRepository(repository)
+	if project != "" {
+		t.Errorf("unexpected project: [%s] != [%s]", project, "")
+	}
+
+	if rest != "" {
+		t.Errorf("unexpected rest: [%s] != [%s]", rest, "")
+	}
+}