diff --git a/api/base.go b/api/base.go index 72f9da50b..8121cc79c 100644 --- a/api/base.go +++ b/api/base.go @@ -31,6 +31,11 @@ import ( "github.com/astaxie/beego" ) +const ( + defaultPageSize int64 = 10 + maxPageSize int64 = 100 +) + // BaseAPI wraps common methods for controllers to host API type BaseAPI struct { beego.Controller @@ -138,6 +143,60 @@ func (b *BaseAPI) GetIDFromURL() int64 { return id } +// set "Link" and "X-Total-Count" header for pagination request +func (b *BaseAPI) setPaginationHeader(total, page, pageSize int64) { + b.Ctx.ResponseWriter.Header().Set("X-Total-Count", strconv.FormatInt(total, 10)) + + link := "" + + // set previous link + if page > 1 && (page-1)*pageSize <= total { + u := *(b.Ctx.Request.URL) + q := u.Query() + q.Set("page", strconv.FormatInt(page-1, 10)) + u.RawQuery = q.Encode() + if len(link) != 0 { + link += ", " + } + link += fmt.Sprintf("<%s>; rel=\"prev\"", u.String()) + } + + // set next link + if pageSize*page < total { + u := *(b.Ctx.Request.URL) + q := u.Query() + q.Set("page", strconv.FormatInt(page+1, 10)) + u.RawQuery = q.Encode() + if len(link) != 0 { + link += ", " + } + link += fmt.Sprintf("<%s>; rel=\"next\"", u.String()) + } + + if len(link) != 0 { + b.Ctx.ResponseWriter.Header().Set("Link", link) + } +} + +func (b *BaseAPI) getPaginationParams() (page, pageSize int64) { + page, err := b.GetInt64("page", 1) + if err != nil || page <= 0 { + b.CustomAbort(http.StatusBadRequest, "invalid page") + } + + pageSize, err = b.GetInt64("page_size", defaultPageSize) + if err != nil || pageSize <= 0 { + b.CustomAbort(http.StatusBadRequest, "invalid page_size") + } + + if pageSize > maxPageSize { + pageSize = maxPageSize + log.Debugf("the parameter page_size %d exceeds the max %d, set it to max", pageSize, maxPageSize) + } + + return page, pageSize +} + func getIsInsecure() bool { insecure := false diff --git a/api/project.go b/api/project.go index 642d71f24..8ea705aa5 100644 --- a/api/project.go +++ b/api/project.go @@ -238,25 +238,25 @@ func (p *ProjectAPI) ToggleProjectPublic() { func (p *ProjectAPI) FilterAccessLog() { p.userID = p.ValidateUser() - var filter models.AccessLog - p.DecodeJSONReq(&filter) + var query models.AccessLog + p.DecodeJSONReq(&query) - username := filter.Username - keywords := filter.Keywords + query.ProjectID = p.projectID + query.Username = "%" + query.Username + "%" + query.BeginTime = time.Unix(query.BeginTimestamp, 0) + query.EndTime = time.Unix(query.EndTimestamp, 0) - beginTime := time.Unix(filter.BeginTimestamp, 0) - endTime := time.Unix(filter.EndTimestamp, 0) + page, pageSize := p.getPaginationParams() - query := models.AccessLog{ProjectID: p.projectID, Username: "%" + username + "%", Keywords: keywords, BeginTime: beginTime, BeginTimestamp: filter.BeginTimestamp, EndTime: endTime, EndTimestamp: filter.EndTimestamp} - - log.Infof("Query AccessLog: begin: %v, end: %v, keywords: %s", query.BeginTime, query.EndTime, query.Keywords) - - accessLogList, err := dao.GetAccessLogs(query) + logs, total, err := dao.GetAccessLogs(query, pageSize, pageSize*(page-1)) if err != nil { - log.Errorf("Error occurred in GetAccessLogs, error: %v", err) - p.CustomAbort(http.StatusInternalServerError, "Internal error.") + log.Errorf("failed to get access log: %v", err) + p.CustomAbort(http.StatusInternalServerError, "") } - p.Data["json"] = accessLogList + + p.setPaginationHeader(total, page, pageSize) + + p.Data["json"] = logs p.ServeJSON() } diff --git a/api/replication_job.go b/api/replication_job.go index ae3d080a3..bc1fced44 100644 --- a/api/replication_job.go +++ b/api/replication_job.go @@ -56,43 +56,28 @@ func (ra *RepJobAPI) Prepare() { } -// List filters jobs according to the policy and repository +// List filters jobs according to the parameters func (ra *RepJobAPI) List() { - var policyID int64 - var repository, status string - var startTime, endTime *time.Time - var num int - var err error - policyIDStr := ra.GetString("policy_id") - if len(policyIDStr) != 0 { - policyID, err = strconv.ParseInt(policyIDStr, 10, 64) - if err != nil || policyID <= 0 { - ra.CustomAbort(http.StatusBadRequest, fmt.Sprintf("invalid policy ID: %s", policyIDStr)) - } + policyID, err := ra.GetInt64("policy_id") + if err != nil || policyID <= 0 { + ra.CustomAbort(http.StatusBadRequest, "invalid policy_id") } - numStr := ra.GetString("num") - if len(numStr) != 0 { - num, err = strconv.Atoi(numStr) - if err != nil { - ra.CustomAbort(http.StatusBadRequest, fmt.Sprintf("invalid num: %s", numStr)) - } - } - if num <= 0 { - num = 200 + policy, err := dao.GetRepPolicy(policyID) + if err != nil { + log.Errorf("failed to get policy %d: %v", policyID, err) + ra.CustomAbort(http.StatusInternalServerError, "") } - endTimeStr := ra.GetString("end_time") - if len(endTimeStr) != 0 { - i, err := strconv.ParseInt(endTimeStr, 10, 64) - if err != nil { - ra.CustomAbort(http.StatusBadRequest, "invalid end_time") - } - t := time.Unix(i, 0) - endTime = &t + if policy == nil { + ra.CustomAbort(http.StatusNotFound, fmt.Sprintf("policy %d not found", policyID)) } + repository := ra.GetString("repository") + status := ra.GetString("status") + + var startTime *time.Time startTimeStr := ra.GetString("start_time") if len(startTimeStr) != 0 { i, err := strconv.ParseInt(startTimeStr, 10, 64) @@ -103,21 +88,29 @@ func (ra *RepJobAPI) List() { startTime = &t } - if startTime == nil && endTime == nil { - // if start_time and end_time are both null, list jobs of last 10 days - t := time.Now().UTC().AddDate(0, 0, -10) - startTime = &t + var endTime *time.Time + endTimeStr := ra.GetString("end_time") + if len(endTimeStr) != 0 { + i, err := strconv.ParseInt(endTimeStr, 10, 64) + if err != nil { + ra.CustomAbort(http.StatusBadRequest, "invalid end_time") + } + t := time.Unix(i, 0) + endTime = &t } - repository = ra.GetString("repository") - status = ra.GetString("status") + page, pageSize := ra.getPaginationParams() - jobs, err := dao.FilterRepJobs(policyID, repository, status, startTime, endTime, num) + jobs, total, err := dao.FilterRepJobs(policyID, repository, status, + startTime, endTime, pageSize, pageSize*(page-1)) if err != nil { - log.Errorf("failed to filter jobs according policy ID %d, repository %s, status %s: %v", policyID, repository, status, err) - ra.RenderError(http.StatusInternalServerError, "Failed to query job") - return + log.Errorf("failed to filter jobs according policy ID %d, repository %s, status %s, start time %v, end time %v: %v", + policyID, repository, status, startTime, endTime, err) + ra.CustomAbort(http.StatusInternalServerError, "") } + + ra.setPaginationHeader(total, page, pageSize) + ra.Data["json"] = jobs ra.ServeJSON() } diff --git a/api/repository.go b/api/repository.go index 8de2f2bcc..da00d691d 100644 --- a/api/repository.go +++ b/api/repository.go @@ -17,6 +17,7 @@ package api import ( "encoding/json" + "fmt" "net/http" "os" "sort" @@ -34,6 +35,7 @@ import ( registry_error "github.com/vmware/harbor/utils/registry/error" + "github.com/vmware/harbor/utils" "github.com/vmware/harbor/utils/registry/auth" ) @@ -46,23 +48,23 @@ type RepositoryAPI struct { // Get ... func (ra *RepositoryAPI) Get() { projectID, err := ra.GetInt64("project_id") - if err != nil { - log.Errorf("Failed to get project id, error: %v", err) - ra.RenderError(http.StatusBadRequest, "Invalid project id") - return - } - p, err := dao.GetProjectByID(projectID) - if err != nil { - log.Errorf("Error occurred in GetProjectById, error: %v", err) - ra.CustomAbort(http.StatusInternalServerError, "Internal error.") - } - if p == nil { - log.Warningf("Project with Id: %d does not exist", projectID) - ra.RenderError(http.StatusNotFound, "") - return + if err != nil || projectID <= 0 { + ra.CustomAbort(http.StatusBadRequest, "invalid project_id") } - if p.Public == 0 { + page, pageSize := ra.getPaginationParams() + + project, err := dao.GetProjectByID(projectID) + if err != nil { + log.Errorf("failed to get project %d: %v", projectID, err) + ra.CustomAbort(http.StatusInternalServerError, "") + } + + if project == nil { + ra.CustomAbort(http.StatusNotFound, fmt.Sprintf("project %d not found", projectID)) + } + + if project.Public == 0 { var userID int if svc_utils.VerifySecret(ra.Ctx.Request) { @@ -72,37 +74,47 @@ func (ra *RepositoryAPI) Get() { } if !checkProjectPermission(userID, projectID) { - ra.RenderError(http.StatusForbidden, "") - return + ra.CustomAbort(http.StatusForbidden, "") } } repoList, err := cache.GetRepoFromCache() if err != nil { - log.Errorf("Failed to get repo from cache, error: %v", err) - ra.RenderError(http.StatusInternalServerError, "internal sever error") + log.Errorf("failed to get repository from cache: %v", err) + ra.CustomAbort(http.StatusInternalServerError, "") } - projectName := p.Name + repositories := []string{} + q := ra.GetString("q") - var resp []string - if len(q) > 0 { - for _, r := range repoList { - if strings.Contains(r, "/") && strings.Contains(r[strings.LastIndex(r, "/")+1:], q) && r[0:strings.LastIndex(r, "/")] == projectName { - resp = append(resp, r) - } + for _, repo := range repoList { + pn, rest := utils.ParseRepository(repo) + if project.Name != pn { + continue } - ra.Data["json"] = resp - } else if len(projectName) > 0 { - for _, r := range repoList { - if strings.Contains(r, "/") && r[0:strings.LastIndex(r, "/")] == projectName { - resp = append(resp, r) - } + + if len(q) != 0 && !strings.Contains(rest, q) { + continue } - ra.Data["json"] = resp - } else { - ra.Data["json"] = repoList + + repositories = append(repositories, repo) } + + total := int64(len(repositories)) + + if (page-1)*pageSize > total { + repositories = []string{} + } else { + repositories = repositories[(page-1)*pageSize:] + } + + if page*pageSize <= total { + repositories = repositories[:pageSize] + } + + ra.setPaginationHeader(total, page, pageSize) + + ra.Data["json"] = repositories ra.ServeJSON() } diff --git a/dao/accesslog.go b/dao/accesslog.go index 930607701..b76a40072 100644 --- a/dao/accesslog.go +++ b/dao/accesslog.go @@ -39,67 +39,78 @@ func AddAccessLog(accessLog models.AccessLog) error { } //GetAccessLogs gets access logs according to different conditions -func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) { - +func GetAccessLogs(query models.AccessLog, limit, offset int64) ([]models.AccessLog, int64, error) { o := GetOrmer() - sql := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time - from access_log a left join user u on a.user_id = u.user_id + + condition := ` from access_log a left join user u on a.user_id = u.user_id where a.project_id = ? ` queryParam := make([]interface{}, 1) - queryParam = append(queryParam, accessLog.ProjectID) + queryParam = append(queryParam, query.ProjectID) - if accessLog.UserID != 0 { - sql += ` and a.user_id = ? ` - queryParam = append(queryParam, accessLog.UserID) + if query.UserID != 0 { + condition += ` and a.user_id = ? ` + queryParam = append(queryParam, query.UserID) } - if accessLog.Operation != "" { - sql += ` and a.operation = ? ` - queryParam = append(queryParam, accessLog.Operation) + if query.Operation != "" { + condition += ` and a.operation = ? ` + queryParam = append(queryParam, query.Operation) } - if accessLog.Username != "" { - sql += ` and u.username like ? ` - queryParam = append(queryParam, accessLog.Username) + if query.Username != "" { + condition += ` and u.username like ? ` + queryParam = append(queryParam, query.Username) } - if accessLog.RepoName != "" { - sql += ` and a.repo_name = ? ` - queryParam = append(queryParam, accessLog.RepoName) + if query.RepoName != "" { + condition += ` and a.repo_name = ? ` + queryParam = append(queryParam, query.RepoName) } - if accessLog.RepoTag != "" { - sql += ` and a.repo_tag = ? ` - queryParam = append(queryParam, accessLog.RepoTag) + if query.RepoTag != "" { + condition += ` and a.repo_tag = ? ` + queryParam = append(queryParam, query.RepoTag) } - if accessLog.Keywords != "" { - sql += ` and a.operation in ( ` - keywordList := strings.Split(accessLog.Keywords, "/") + if query.Keywords != "" { + condition += ` and a.operation in ( ` + keywordList := strings.Split(query.Keywords, "/") num := len(keywordList) for i := 0; i < num; i++ { if keywordList[i] != "" { if i == num-1 { - sql += `?)` + condition += `?)` } else { - sql += `?,` + condition += `?,` } queryParam = append(queryParam, keywordList[i]) } } } - if accessLog.BeginTimestamp > 0 { - sql += ` and a.op_time >= ? ` - queryParam = append(queryParam, accessLog.BeginTime) + if query.BeginTimestamp > 0 { + condition += ` and a.op_time >= ? ` + queryParam = append(queryParam, query.BeginTime) } - if accessLog.EndTimestamp > 0 { - sql += ` and a.op_time <= ? ` - queryParam = append(queryParam, accessLog.EndTime) + if query.EndTimestamp > 0 { + condition += ` and a.op_time <= ? ` + queryParam = append(queryParam, query.EndTime) } - sql += ` order by a.op_time desc ` + condition += ` order by a.op_time desc ` - var accessLogList []models.AccessLog - _, err := o.Raw(sql, queryParam).QueryRows(&accessLogList) + totalSQL := `select count(*) ` + condition + + logs := []models.AccessLog{} + + var total int64 + if err := o.Raw(totalSQL, queryParam).QueryRow(&total); err != nil { + return logs, 0, err + } + + condition = paginateForRawSQL(condition, limit, offset) + + recordsSQL := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time ` + condition + _, err := o.Raw(recordsSQL, queryParam).QueryRows(&logs) if err != nil { - return nil, err + return logs, 0, err } - return accessLogList, nil + + return logs, total, nil } // AccessLog ... diff --git a/dao/base.go b/dao/base.go index 97bc55ad0..0ac408c41 100644 --- a/dao/base.go +++ b/dao/base.go @@ -16,6 +16,7 @@ package dao import ( + "fmt" "net" "os" @@ -44,7 +45,7 @@ func GenerateRandomString() (string, error) { //InitDB initializes the database func InitDB() { - // orm.Debug = true + // orm.Debug = true orm.RegisterDriver("mysql", orm.DRMySQL) addr := os.Getenv("MYSQL_HOST") port := os.Getenv("MYSQL_PORT") @@ -89,3 +90,7 @@ func GetOrmer() orm.Ormer { }) return globalOrm } + +func paginateForRawSQL(sql string, limit, offset int64) string { + return fmt.Sprintf("%s limit %d offset %d", sql, limit, offset) +} diff --git a/dao/dao_test.go b/dao/dao_test.go index 3d375927d..52ea37a27 100644 --- a/dao/dao_test.go +++ b/dao/dao_test.go @@ -420,7 +420,7 @@ func TestGetAccessLog(t *testing.T) { UserID: currentUser.UserID, ProjectID: currentProject.ProjectID, } - accessLogs, err := GetAccessLogs(queryAccessLog) + accessLogs, _, err := GetAccessLogs(queryAccessLog, 1000, 0) if err != nil { t.Errorf("Error occurred in GetAccessLog: %v", err) } @@ -448,7 +448,7 @@ func TestAddAccessLog(t *testing.T) { if err != nil { t.Errorf("Error occurred in AddAccessLog: %v", err) } - accessLogList, err = GetAccessLogs(accessLog) + accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0) if err != nil { t.Errorf("Error occurred in GetAccessLog: %v", err) } @@ -477,7 +477,7 @@ func TestAccessLog(t *testing.T) { if err != nil { t.Errorf("Error occurred in AccessLog: %v", err) } - accessLogList, err = GetAccessLogs(accessLog) + accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0) if err != nil { t.Errorf("Error occurred in GetAccessLog: %v", err) } @@ -1178,7 +1178,7 @@ func TestGetRepJobByPolicy(t *testing.T) { } func TestFilterRepJobs(t *testing.T) { - jobs, err := FilterRepJobs(policyID, "", "", nil, nil, 1000) + jobs, _, err := FilterRepJobs(policyID, "", "", nil, nil, 1000, 0) if err != nil { t.Errorf("Error occured in FilterRepJobs: %v, policy ID: %d", err, policyID) return diff --git a/dao/replication_job.go b/dao/replication_job.go index 38e155bc2..5e9f8324e 100644 --- a/dao/replication_job.go +++ b/dao/replication_job.go @@ -312,12 +312,14 @@ func GetRepJobByPolicy(policyID int64) ([]*models.RepJob, error) { return res, err } -// FilterRepJobs filters jobs by repo and policy ID +// FilterRepJobs ... func FilterRepJobs(policyID int64, repository, status string, startTime, - endTime *time.Time, limit int) ([]*models.RepJob, error) { - o := GetOrmer() + endTime *time.Time, limit, offset int64) ([]*models.RepJob, int64, error) { + + jobs := []*models.RepJob{} + + qs := GetOrmer().QueryTable(new(models.RepJob)) - qs := o.QueryTable(new(models.RepJob)) if policyID != 0 { qs = qs.Filter("PolicyID", policyID) } @@ -327,32 +329,28 @@ func FilterRepJobs(policyID int64, repository, status string, startTime, if len(status) != 0 { qs = qs.Filter("Status__icontains", status) } - if startTime != nil { - fmt.Printf("%v\n", startTime) qs = qs.Filter("CreationTime__gte", startTime) } - if endTime != nil { - fmt.Printf("%v\n", endTime) qs = qs.Filter("CreationTime__lte", endTime) } - if limit != 0 { - qs = qs.Limit(limit) - } - qs = qs.OrderBy("-UpdateTime") - var jobs []*models.RepJob - _, err := qs.All(&jobs) + total, err := qs.Count() if err != nil { - return nil, err + return jobs, 0, err + } + + _, err = qs.Limit(limit).Offset(offset).All(&jobs) + if err != nil { + return jobs, 0, err } genTagListForJob(jobs...) - return jobs, nil + return jobs, total, nil } // GetRepJobToStop get jobs that are possibly being handled by workers of a certain policy. diff --git a/service/utils/utils.go b/service/utils/utils.go index ae54bc3d3..fb441f6dd 100644 --- a/service/utils/utils.go +++ b/service/utils/utils.go @@ -17,9 +17,10 @@ package utils import ( - "github.com/vmware/harbor/utils/log" "net/http" "os" + + "github.com/vmware/harbor/utils/log" ) // VerifySecret verifies the UI_SECRET cookie in a http request. @@ -27,7 +28,7 @@ func VerifySecret(r *http.Request) bool { secret := os.Getenv("UI_SECRET") c, err := r.Cookie("uisecret") if err != nil { - log.Errorf("Failed to get secret cookie, error: %v", err) + log.Warningf("Failed to get secret cookie, error: %v", err) } return c != nil && c.Value == secret } diff --git a/utils/utils.go b/utils/utils.go index 653e5409c..7fa1215e4 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -27,10 +27,8 @@ type Repository struct { // GetProject parses the repository and return the name of project. func (r *Repository) GetProject() string { - if !strings.ContainsRune(r.Name, '/') { - return "" - } - return r.Name[0:strings.LastIndex(r.Name, "/")] + project, _ := ParseRepository(r.Name) + return project } // FormatEndpoint formats endpoint @@ -55,3 +53,17 @@ func ParseEndpoint(endpoint string) (*url.URL, error) { } return u, nil } + +// ParseRepository splits a repository into two parts: project and rest +func ParseRepository(repository string) (project, rest string) { + repository = strings.TrimLeft(repository, "/") + repository = strings.TrimRight(repository, "/") + if !strings.ContainsRune(repository, '/') { + rest = repository + return + } + index := strings.LastIndex(repository, "/") + project = repository[0:index] + rest = repository[index+1:] + return +} diff --git a/utils/utils_test.go b/utils/utils_test.go index 7d2394686..f2b23c1da 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -1,3 +1,18 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package utils import ( @@ -7,3 +22,42 @@ import ( func TestMain(t *testing.T) { } +func TestParseRepository(t *testing.T) { + repository := "library/ubuntu" + project, rest := ParseRepository(repository) + if project != "library" { + t.Errorf("unexpected project: %s != %s", project, "library") + } + if rest != "ubuntu" { + t.Errorf("unexpected rest: %s != %s", rest, "ubuntu") + } + + repository = "library/test/ubuntu" + project, rest = ParseRepository(repository) + if project != "library/test" { + t.Errorf("unexpected project: %s != %s", project, "library/test") + } + if rest != "ubuntu" { + t.Errorf("unexpected rest: %s != %s", rest, "ubuntu") + } + + repository = "ubuntu" + project, rest = ParseRepository(repository) + if project != "" { + t.Errorf("unexpected project: [%s] != [%s]", project, "") + } + + if rest != "ubuntu" { + t.Errorf("unexpected rest: %s != %s", rest, "ubuntu") + } + + repository = "" + project, rest = ParseRepository(repository) + if project != "" { + t.Errorf("unexpected project: [%s] != [%s]", project, "") + } + + if rest != "" { + t.Errorf("unexpected rest: [%s] != [%s]", rest, "") + } +}