From e625f2aa118def0477a0f976c00d6fe7723bc508 Mon Sep 17 00:00:00 2001 From: He Weiwei Date: Thu, 25 Jul 2019 13:40:26 +0800 Subject: [PATCH] feat(quota,api): APIs for quotas Signed-off-by: He Weiwei --- docs/swagger.yaml | 200 +++++++++++++++++++ src/common/dao/project/projectmember.go | 47 +++-- src/common/dao/project/projectmember_test.go | 44 +++- src/common/dao/quota.go | 137 ++++++++++--- src/common/models/project.go | 18 ++ src/common/models/quota.go | 8 + src/common/quota/manager.go | 2 + src/common/utils/utils.go | 9 +- src/common/utils/utils_test.go | 28 +++ src/core/api/harborapi_test.go | 74 +++++++ src/core/api/project.go | 76 +++++++ src/core/api/project_test.go | 63 ++++++ src/core/api/quota.go | 155 ++++++++++++++ src/core/api/quota_test.go | 133 ++++++++++++ src/core/router.go | 4 + src/testing/apitests/apilib/project.go | 16 ++ src/testing/apitests/apilib/quota.go | 39 ++++ 17 files changed, 1014 insertions(+), 39 deletions(-) create mode 100644 src/core/api/quota.go create mode 100644 src/core/api/quota_test.go create mode 100644 src/testing/apitests/apilib/quota.go diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 1d39c7bb5..b8e16e8ed 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -311,6 +311,34 @@ paths: description: User need to log in first. '500': description: Unexpected internal errors. + '/projects/{project_id}/summary': + get: + summary: Get summary of the project. + description: Get summary of the project. + parameters: + - name: project_id + in: path + type: integer + format: int64 + required: true + description: Relevant project ID + tags: + - Products + responses: + '200': + description: Get summary of the project successfully. + schema: + $ref: '#/definitions/ProjectSummary' + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '404': + description: Project ID does not exist. + '403': + description: User does not have permission to get summary of the project. + '500': + description: Unexpected internal errors. '/projects/{project_id}/metadatas': get: summary: Get project metadata. @@ -3547,6 +3575,113 @@ paths: description: User does not have permission to call this API. '500': description: Unexpected internal errors. + '/quotas': + get: + summary: List quotas + description: List quotas + tags: + - quota + parameters: + - name: reference + in: query + description: The reference type of quota. + required: false + type: string + - name: sort + in: query + type: string + required: false + description: | + Sort method, valid values include: + 'hard.resource_name', '-hard.resource_name', 'used.resource_name', '-used.resource_name'. + Here '-' stands for descending order, resource_name should be the real resource name of the quota. + - name: page + in: query + type: integer + format: int32 + required: false + description: 'The page nubmer, default is 1.' + - name: page_size + in: query + type: integer + format: int32 + required: false + description: 'The size of per page, default is 10, maximum is 100.' + responses: + '200': + description: Successfully retrieved the quotas. + schema: + type: array + items: + $ref: '#/definitions/Quota' + headers: + X-Total-Count: + description: The total count of access logs + type: integer + Link: + description: Link refers to the previous page and next page + type: string + '401': + description: User is not authenticated. + '403': + description: User does not have permission to call this API. + '500': + description: Unexpected internal errors. + '/quotas/{id}': + get: + summary: Get the specified quota + description: Get the specified quota + tags: + - quota + parameters: + - name: id + in: path + type: integer + required: true + description: Quota ID + responses: + '200': + description: Successfully retrieved the quota. + schema: + $ref: '#/definitions/Quota' + '401': + description: User need to log in first. + '403': + description: User does not have permission to call this API + '404': + description: Quota does not exist. + '500': + description: Unexpected internal errors. + put: + summary: Update the specified quota + description: Update hard limits of the specified quota + tags: + - quota + parameters: + - name: id + in: path + type: integer + required: true + description: Quota ID + - name: hard + in: body + required: true + description: The new hard limits for the quota + schema: + $ref: '#/definitions/QuotaUpdateReq' + responses: + '200': + description: Updated quota hard limits successfully. + '400': + description: Illegal format of quota update request. + '401': + description: User need to log in first. + '403': + description: User does not have permission to the quota. + '404': + description: Quota ID does not exist. + '500': + description: Unexpected internal errors. responses: OK: description: 'Success' @@ -3699,6 +3834,36 @@ definitions: auto_scan: type: string description: 'Whether scan images automatically when pushing. The valid values are "true", "false".' + ProjectSummary: + type: object + properties: + repo_count: + type: integer + description: The number of the repositories under this project. + chart_count: + type: integer + description: The total number of charts under this project. + project_admin_count: + type: integer + description: The total number of project admin members. + master_count: + type: integer + description: The total number of master members. + developer_count: + type: integer + description: The total number of developer members. + guest_count: + type: integer + description: The total number of guest members. + quota: + type: object + properties: + hard: + $ref: "#/definitions/ResourceList" + description: The hard limits of the quota + used: + $ref: "#/definitions/ResourceList" + description: The used status of the quota Manifest: type: object properties: @@ -5186,3 +5351,38 @@ definitions: cve_id: type: string description: The ID of the CVE, such as "CVE-2019-10164" + ResourceList: + type: object + additionalProperties: + type: integer + QuotaUpdateReq: + type: object + properties: + hard: + $ref: "#/definitions/ResourceList" + description: The new hard limits for the quota + QuotaRefObject: + type: object + additionalProperties: {} + Quota: + type: object + description: The quota object + properties: + id: + type: integer + description: ID of the quota + ref: + $ref: "#/definitions/QuotaRefObject" + description: The reference object of the quota + hard: + $ref: "#/definitions/ResourceList" + description: The hard limits of the quota + used: + $ref: "#/definitions/ResourceList" + description: The used status of the quota + creation_time: + type: string + description: the creation time of the quota + update_time: + type: string + description: the update time of the quota \ No newline at end of file diff --git a/src/common/dao/project/projectmember.go b/src/common/dao/project/projectmember.go index 6776143ad..081b036f0 100644 --- a/src/common/dao/project/projectmember.go +++ b/src/common/dao/project/projectmember.go @@ -30,13 +30,13 @@ func GetProjectMember(queryMember models.Member) ([]*models.Member, error) { } o := dao.GetOrmer() - sql := ` select a.* from (select pm.id as id, pm.project_id as project_id, ug.id as entity_id, ug.group_name as entity_name, ug.creation_time, ug.update_time, r.name as rolename, - r.role_id as role, pm.entity_type as entity_type from user_group ug join project_member pm + sql := ` select a.* from (select pm.id as id, pm.project_id as project_id, ug.id as entity_id, ug.group_name as entity_name, ug.creation_time, ug.update_time, r.name as rolename, + r.role_id as role, pm.entity_type as entity_type from user_group ug join project_member pm on pm.project_id = ? and ug.id = pm.entity_id join role r on pm.role = r.role_id where pm.entity_type = 'g' union - select pm.id as id, pm.project_id as project_id, u.user_id as entity_id, u.username as entity_name, u.creation_time, u.update_time, r.name as rolename, - r.role_id as role, pm.entity_type as entity_type from harbor_user u join project_member pm - on pm.project_id = ? and u.user_id = pm.entity_id + select pm.id as id, pm.project_id as project_id, u.user_id as entity_id, u.username as entity_name, u.creation_time, u.update_time, r.name as rolename, + r.role_id as role, pm.entity_type as entity_type from harbor_user u join project_member pm + on pm.project_id = ? and u.user_id = pm.entity_id join role r on pm.role = r.role_id where u.deleted = false and pm.entity_type = 'u') as a where a.project_id = ? ` queryParam := make([]interface{}, 1) @@ -70,6 +70,27 @@ func GetProjectMember(queryMember models.Member) ([]*models.Member, error) { return members, err } +// GetTotalOfProjectMembers returns total of project members +func GetTotalOfProjectMembers(projectID int64, roles ...int) (int64, error) { + log.Debugf("Query condition %+v", projectID) + if projectID == 0 { + return 0, fmt.Errorf("failed to get total of project members, project id required %v", projectID) + } + + sql := "SELECT COUNT(1) FROM project_member WHERE project_id = ?" + + queryParam := []interface{}{projectID} + + if len(roles) > 0 { + sql += " AND role = ?" + queryParam = append(queryParam, roles[0]) + } + + var count int64 + err := dao.GetOrmer().Raw(sql, queryParam).QueryRow(&count) + return count, err +} + // AddProjectMember inserts a record to table project_member func AddProjectMember(member models.Member) (int, error) { @@ -120,23 +141,23 @@ func DeleteProjectMemberByID(pmid int) error { // SearchMemberByName search members of the project by entity_name func SearchMemberByName(projectID int64, entityName string) ([]*models.Member, error) { o := dao.GetOrmer() - sql := `select pm.id, pm.project_id, - u.username as entity_name, + sql := `select pm.id, pm.project_id, + u.username as entity_name, r.name as rolename, - pm.role, pm.entity_id, pm.entity_type + pm.role, pm.entity_id, pm.entity_type from project_member pm left join harbor_user u on pm.entity_id = u.user_id and pm.entity_type = 'u' left join role r on pm.role = r.role_id - where u.deleted = false and pm.project_id = ? and u.username like ? + where u.deleted = false and pm.project_id = ? and u.username like ? union - select pm.id, pm.project_id, - ug.group_name as entity_name, + select pm.id, pm.project_id, + ug.group_name as entity_name, r.name as rolename, - pm.role, pm.entity_id, pm.entity_type + pm.role, pm.entity_id, pm.entity_type from project_member pm left join user_group ug on pm.entity_id = ug.id and pm.entity_type = 'g' left join role r on pm.role = r.role_id - where pm.project_id = ? and ug.group_name like ? + where pm.project_id = ? and ug.group_name like ? order by entity_name ` queryParam := make([]interface{}, 4) queryParam = append(queryParam, projectID) diff --git a/src/common/dao/project/projectmember_test.go b/src/common/dao/project/projectmember_test.go index f19738009..fadb598b2 100644 --- a/src/common/dao/project/projectmember_test.go +++ b/src/common/dao/project/projectmember_test.go @@ -51,11 +51,18 @@ func TestMain(m *testing.M) { "update project set owner_id = (select user_id from harbor_user where username = 'member_test_01') where name = 'member_test_01'", "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_01') , (select user_id from harbor_user where username = 'member_test_01'), 'u', 1)", "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_01') , (select id from user_group where group_name = 'test_group_01'), 'g', 1)", + + "insert into harbor_user (username, email, password, realname) values ('member_test_02', 'member_test_02@example.com', '123456', 'member_test_02')", + "insert into project (name, owner_id) values ('member_test_02', 1)", + "insert into user_group (group_name, group_type, ldap_group_dn) values ('test_group_02', 1, 'CN=harbor_users,OU=sample,OU=vmware,DC=harbor,DC=com')", + "update project set owner_id = (select user_id from harbor_user where username = 'member_test_02') where name = 'member_test_02'", + "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_02') , (select user_id from harbor_user where username = 'member_test_02'), 'u', 1)", + "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_02') , (select id from user_group where group_name = 'test_group_02'), 'g', 1)", } clearSqls := []string{ - "delete from project where name='member_test_01'", - "delete from harbor_user where username='member_test_01' or username='pm_sample'", + "delete from project where name='member_test_01' or name='member_test_02'", + "delete from harbor_user where username='member_test_01' or username='member_test_02' or username='pm_sample'", "delete from user_group", "delete from project_member", } @@ -285,6 +292,39 @@ func TestGetProjectMember(t *testing.T) { } } + +func TestGetTotalOfProjectMembers(t *testing.T) { + currentProject, _ := dao.GetProjectByName("member_test_02") + + type args struct { + projectID int64 + roles []int + } + tests := []struct { + name string + args args + want int64 + wantErr bool + }{ + {"Get total of project admin", args{currentProject.ProjectID, []int{common.RoleProjectAdmin}}, 2, false}, + {"Get total of master", args{currentProject.ProjectID, []int{common.RoleMaster}}, 0, false}, + {"Get total of developer", args{currentProject.ProjectID, []int{common.RoleDeveloper}}, 0, false}, + {"Get total of guest", args{currentProject.ProjectID, []int{common.RoleGuest}}, 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetTotalOfProjectMembers(tt.args.projectID, tt.args.roles...) + if (err != nil) != tt.wantErr { + t.Errorf("GetTotalOfProjectMembers() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetTotalOfProjectMembers() = %v, want %v", got, tt.want) + } + }) + } +} + func PrepareGroupTest() { initSqls := []string{ `insert into user_group (group_name, group_type, ldap_group_dn) values ('harbor_group_01', 1, 'cn=harbor_user,dc=example,dc=com')`, diff --git a/src/common/dao/quota.go b/src/common/dao/quota.go index 4252e51e6..6cf130d3d 100644 --- a/src/common/dao/quota.go +++ b/src/common/dao/quota.go @@ -15,25 +15,26 @@ package dao import ( + "encoding/json" "fmt" "strings" "time" "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota/driver" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/types" ) var ( quotaOrderMap = map[string]string{ - "id": "id asc", - "+id": "id asc", - "-id": "id desc", - "creation_time": "creation_time asc", - "+creation_time": "creation_time asc", - "-creation_time": "creation_time desc", - "update_time": "update_time asc", - "+update_time": "update_time asc", - "-update_time": "update_time desc", + "creation_time": "b.creation_time asc", + "+creation_time": "b.creation_time asc", + "-creation_time": "b.creation_time desc", + "update_time": "b.update_time asc", + "+update_time": "b.update_time asc", + "-update_time": "b.update_time desc", } ) @@ -62,10 +63,58 @@ func UpdateQuota(quota models.Quota) error { return err } +// Quota quota mode for api +type Quota struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Ref driver.RefObject `json:"ref"` + Reference string `orm:"column(reference)" json:"-"` + ReferenceID string `orm:"column(reference_id)" json:"-"` + Hard string `orm:"column(hard);type(jsonb)" json:"-"` + Used string `orm:"column(used);type(jsonb)" json:"-"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +// MarshalJSON ... +func (q *Quota) MarshalJSON() ([]byte, error) { + hard, err := types.NewResourceList(q.Hard) + if err != nil { + return nil, err + } + + used, err := types.NewResourceList(q.Used) + if err != nil { + return nil, err + } + + type Alias Quota + return json.Marshal(&struct { + *Alias + Hard types.ResourceList `json:"hard"` + Used types.ResourceList `json:"used"` + }{ + Alias: (*Alias)(q), + Hard: hard, + Used: used, + }) +} + // ListQuotas returns quotas by query. -func ListQuotas(query ...*models.QuotaQuery) ([]*models.Quota, error) { +func ListQuotas(query ...*models.QuotaQuery) ([]*Quota, error) { condition, params := quotaQueryConditions(query...) - sql := fmt.Sprintf(`select * %s`, condition) + + sql := fmt.Sprintf(` +SELECT + a.id, + a.reference, + a.reference_id, + a.hard, + b.used, + b.creation_time, + b.update_time +FROM + quota AS a + JOIN quota_usage AS b ON a.id = b.id %s`, condition) orderBy := quotaOrderBy(query...) if orderBy != "" { @@ -84,42 +133,80 @@ func ListQuotas(query ...*models.QuotaQuery) ([]*models.Quota, error) { } } - var quotas []*models.Quota + var quotas []*Quota if _, err := GetOrmer().Raw(sql, params).QueryRows("as); err != nil { return nil, err } + for _, quota := range quotas { + d, ok := driver.Get(quota.Reference) + if !ok { + continue + } + + ref, err := d.Load(quota.ReferenceID) + if err != nil { + log.Warning(fmt.Sprintf("Load quota reference object (%s, %s) failed: %v", quota.Reference, quota.ReferenceID, err)) + continue + } + + quota.Ref = ref + } + return quotas, nil } +// GetTotalOfQuotas returns total of quotas +func GetTotalOfQuotas(query ...*models.QuotaQuery) (int64, error) { + condition, params := quotaQueryConditions(query...) + sql := fmt.Sprintf("SELECT COUNT(1) FROM quota AS a JOIN quota_usage AS b ON a.id = b.id %s", condition) + + var count int64 + if err := GetOrmer().Raw(sql, params).QueryRow(&count); err != nil { + return 0, err + } + + return count, nil +} + func quotaQueryConditions(query ...*models.QuotaQuery) (string, []interface{}) { params := []interface{}{} - sql := `from quota ` + sql := "" if len(query) == 0 || query[0] == nil { return sql, params } - sql += `where 1=1 ` + sql += `WHERE 1=1 ` q := query[0] + if q.ID != 0 { + sql += `AND a.id = ? ` + params = append(params, q.ID) + } if q.Reference != "" { - sql += `and reference = ? ` + sql += `AND a.reference = ? ` params = append(params, q.Reference) } if q.ReferenceID != "" { - sql += `and reference_id = ? ` + sql += `AND a.reference_id = ? ` params = append(params, q.ReferenceID) } + if len(q.ReferenceIDs) != 0 { - sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs))) + sql += fmt.Sprintf(`AND a.reference_id IN (%s) `, paramPlaceholder(len(q.ReferenceIDs))) params = append(params, q.ReferenceIDs) } return sql, params } +func castQuantity(field string) string { + // cast -1 to max int64 when order by field + return fmt.Sprintf("CAST( (CASE WHEN (%[1]s) IS NULL THEN '0' WHEN (%[1]s) = '-1' THEN '9223372036854775807' ELSE (%[1]s) END) AS BIGINT )", field) +} + func quotaOrderBy(query ...*models.QuotaQuery) string { - orderBy := "" + orderBy := "b.creation_time DESC" if len(query) > 0 && query[0] != nil && query[0].Sort != "" { if val, ok := quotaOrderMap[query[0].Sort]; ok { @@ -127,15 +214,19 @@ func quotaOrderBy(query ...*models.QuotaQuery) string { } else { sort := query[0].Sort - order := "asc" + order := "ASC" if sort[0] == '-' { - order = "desc" + order = "DESC" sort = sort[1:] } - prefix := "hard." - if strings.HasPrefix(sort, prefix) { - orderBy = fmt.Sprintf("hard->>'%s' %s", strings.TrimPrefix(sort, prefix), order) + prefix := []string{"hard.", "used."} + for _, p := range prefix { + if strings.HasPrefix(sort, p) { + field := fmt.Sprintf("%s->>'%s'", strings.TrimSuffix(p, "."), strings.TrimPrefix(sort, p)) + orderBy = fmt.Sprintf("(%s) %s", castQuantity(field), order) + break + } } } } diff --git a/src/common/models/project.go b/src/common/models/project.go index b633ec280..e7f888ae1 100644 --- a/src/common/models/project.go +++ b/src/common/models/project.go @@ -17,6 +17,8 @@ package models import ( "strings" "time" + + "github.com/goharbor/harbor/src/pkg/types" ) // ProjectTable is the table name for project @@ -183,3 +185,19 @@ type ProjectQueryResult struct { func (p *Project) TableName() string { return ProjectTable } + +// ProjectSummary ... +type ProjectSummary struct { + RepoCount int64 `json:"repo_count"` + ChartCount uint64 `json:"chart_count"` + + ProjectAdminCount int64 `json:"project_admin_count"` + MasterCount int64 `json:"master_count"` + DeveloperCount int64 `json:"developer_count"` + GuestCount int64 `json:"guest_count"` + + Quota struct { + Hard types.ResourceList `json:"hard"` + Used types.ResourceList `json:"used"` + } `json:"quota"` +} diff --git a/src/common/models/quota.go b/src/common/models/quota.go index 8e3340272..e7d8ade6e 100644 --- a/src/common/models/quota.go +++ b/src/common/models/quota.go @@ -17,6 +17,8 @@ package models import ( "encoding/json" "time" + + "github.com/goharbor/harbor/src/pkg/types" ) // QuotaHard a map for the quota hard @@ -69,9 +71,15 @@ func (q *Quota) SetHard(hard QuotaHard) { // QuotaQuery query parameters for quota type QuotaQuery struct { + ID int64 Reference string ReferenceID string ReferenceIDs []string Pagination Sorting } + +// QuotaUpdateRequest the request for quota update +type QuotaUpdateRequest struct { + Hard types.ResourceList `json:"hard"` +} diff --git a/src/common/quota/manager.go b/src/common/quota/manager.go index d4eda5d07..43d70777b 100644 --- a/src/common/quota/manager.go +++ b/src/common/quota/manager.go @@ -22,6 +22,7 @@ import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/quota/driver" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/pkg/types" ) @@ -207,6 +208,7 @@ func NewManager(reference string, referenceID string) (*Manager, error) { } if _, err := d.Load(referenceID); err != nil { + log.Warning(fmt.Sprintf("Load quota reference object (%s, %s) failed: %v", reference, referenceID, err)) return nil, err } diff --git a/src/common/utils/utils.go b/src/common/utils/utils.go index cea54d342..d93c3c033 100644 --- a/src/common/utils/utils.go +++ b/src/common/utils/utils.go @@ -230,7 +230,14 @@ func GetStrValueOfAnyType(value interface{}) string { } strVal = string(b) } else { - strVal = fmt.Sprintf("%v", value) + switch val := value.(type) { + case float64: + strVal = strconv.FormatFloat(val, 'f', -1, 64) + case float32: + strVal = strconv.FormatFloat(float64(val), 'f', -1, 32) + default: + strVal = fmt.Sprintf("%v", value) + } } return strVal } diff --git a/src/common/utils/utils_test.go b/src/common/utils/utils_test.go index 66c4bca0f..206124610 100644 --- a/src/common/utils/utils_test.go +++ b/src/common/utils/utils_test.go @@ -381,3 +381,31 @@ func TestTrimLower(t *testing.T) { }) } } + +func TestGetStrValueOfAnyType(t *testing.T) { + type args struct { + value interface{} + } + tests := []struct { + name string + args args + want string + }{ + {"float", args{float32(1048576.1)}, "1048576.1"}, + {"float", args{float64(1048576.12)}, "1048576.12"}, + {"float", args{1048576.000}, "1048576"}, + {"int", args{1048576}, "1048576"}, + {"int", args{9223372036854775807}, "9223372036854775807"}, + {"string", args{"hello world"}, "hello world"}, + {"bool", args{true}, "true"}, + {"bool", args{false}, "false"}, + {"map", args{map[string]interface{}{"key1": "value1"}}, "{\"key1\":\"value1\"}"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetStrValueOfAnyType(tt.args.value); got != tt.want { + t.Errorf("GetStrValueOfAnyType() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index ed51699c8..f2c02f7e2 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -103,6 +103,7 @@ func init() { beego.Router("/api/users/:id/permissions", &UserAPI{}, "get:ListUserPermissions") beego.Router("/api/users/:id/sysadmin", &UserAPI{}, "put:ToggleUserAdminRole") beego.Router("/api/projects/:id([0-9]+)/logs", &ProjectAPI{}, "get:Logs") + beego.Router("/api/projects/:id([0-9]+)/summary", &ProjectAPI{}, "get:Summary") beego.Router("/api/projects/:id([0-9]+)/_deletable", &ProjectAPI{}, "get:Deletable") beego.Router("/api/projects/:id([0-9]+)/metadatas/?:name", &MetadataAPI{}, "get:Get") beego.Router("/api/projects/:id([0-9]+)/metadatas/", &MetadataAPI{}, "post:Post") @@ -180,6 +181,10 @@ func init() { beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels", chartLabelAPIType, "get:GetLabels;post:MarkLabel") beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels/:id([0-9]+)", chartLabelAPIType, "delete:RemoveLabel") + quotaAPIType := &QuotaAPI{} + beego.Router("/api/quotas", quotaAPIType, "get:List") + beego.Router("/api/quotas/:id([0-9]+)", quotaAPIType, "get:Get;put:Put") + // syncRegistry if err := SyncRegistry(config.GlobalProjectMgr); err != nil { log.Fatalf("failed to sync repositories from registry: %v", err) @@ -454,6 +459,23 @@ func (a testapi) ProjectDeletable(prjUsr usrInfo, projectID int64) (int, bool, e return code, deletable.Deletable, nil } +// ProjectSummary returns summary for the project +func (a testapi) ProjectSummary(prjUsr usrInfo, projectID string) (int, apilib.ProjectSummary, error) { + _sling := sling.New().Get(a.basePath) + + // create api path + path := "api/projects/" + projectID + "/summary" + _sling = _sling.Path(path) + + var successPayload apilib.ProjectSummary + + httpStatusCode, body, err := request(_sling, jsonAcceptHeader, prjUsr) + if err == nil && httpStatusCode == 200 { + err = json.Unmarshal(body, &successPayload) + } + return httpStatusCode, successPayload, err +} + // -------------------------Member Test---------------------------------------// // Return relevant role members of projectID @@ -1213,3 +1235,55 @@ func (a testapi) RegistryUpdate(authInfo usrInfo, registryID int64, req *apimode return code, nil } + +// QuotasGet returns quotas +func (a testapi) QuotasGet(query *apilib.QuotaQuery, authInfo ...usrInfo) (int, []apilib.Quota, error) { + _sling := sling.New().Get(a.basePath). + Path("api/quotas"). + QueryStruct(query) + + var successPayload []apilib.Quota + + var httpStatusCode int + var err error + var body []byte + if len(authInfo) > 0 { + httpStatusCode, body, err = request(_sling, jsonAcceptHeader, authInfo[0]) + } else { + httpStatusCode, body, err = request(_sling, jsonAcceptHeader) + } + + if err == nil && httpStatusCode == 200 { + err = json.Unmarshal(body, &successPayload) + } else { + log.Println(string(body)) + } + + return httpStatusCode, successPayload, err +} + +// Return specific quota +func (a testapi) QuotasGetByID(authInfo usrInfo, quotaID string) (int, apilib.Quota, error) { + _sling := sling.New().Get(a.basePath) + + // create api path + path := "api/quotas/" + quotaID + _sling = _sling.Path(path) + + var successPayload apilib.Quota + + httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo) + if err == nil && httpStatusCode == 200 { + err = json.Unmarshal(body, &successPayload) + } + return httpStatusCode, successPayload, err +} + +// Update spec for the quota +func (a testapi) QuotasPut(authInfo usrInfo, quotaID string, req models.QuotaUpdateRequest) (int, error) { + path := "/api/quotas/" + quotaID + _sling := sling.New().Put(a.basePath).Path(path).BodyJSON(req) + + httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo) + return httpStatusCode, err +} diff --git a/src/core/api/project.go b/src/core/api/project.go index cbd3a23a6..953344c2c 100644 --- a/src/core/api/project.go +++ b/src/core/api/project.go @@ -19,10 +19,12 @@ import ( "net/http" "regexp" "strconv" + "sync" "time" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/dao/project" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/quota" "github.com/goharbor/harbor/src/common/rbac" @@ -577,6 +579,33 @@ func (p *ProjectAPI) Logs() { p.ServeJSON() } +// Summary returns the summary of the project +func (p *ProjectAPI) Summary() { + if !p.requireAccess(rbac.ActionRead) { + return + } + + summary := &models.ProjectSummary{ + RepoCount: p.project.RepoCount, + ChartCount: p.project.ChartCount, + } + + var wg sync.WaitGroup + for _, fn := range []func(int64, *models.ProjectSummary){getProjectQuotaSummary, getProjectMemberSummary} { + fn := fn + + wg.Add(1) + go func() { + defer wg.Done() + fn(p.project.ProjectID, summary) + }() + } + wg.Wait() + + p.Data["json"] = summary + p.ServeJSON() +} + // TODO move this to pa ckage models func validateProjectReq(req *models.ProjectRequest) error { pn := req.Name @@ -618,3 +647,50 @@ func projectQuotaHardLimits(req *models.ProjectRequest, setting *models.QuotaSet return hardLimits, nil } + +func getProjectQuotaSummary(projectID int64, summary *models.ProjectSummary) { + quotas, err := dao.ListQuotas(&models.QuotaQuery{Reference: "project", ReferenceID: strconv.FormatInt(projectID, 10)}) + if err != nil { + log.Debugf("failed to get quota for project: %d", projectID) + return + } + + if len(quotas) == 0 { + log.Debugf("quota not found for project: %d", projectID) + return + } + + quota := quotas[0] + + summary.Quota.Hard, _ = types.NewResourceList(quota.Hard) + summary.Quota.Used, _ = types.NewResourceList(quota.Used) +} + +func getProjectMemberSummary(projectID int64, summary *models.ProjectSummary) { + var wg sync.WaitGroup + + for _, e := range []struct { + role int + count *int64 + }{ + {common.RoleProjectAdmin, &summary.ProjectAdminCount}, + {common.RoleMaster, &summary.MasterCount}, + {common.RoleDeveloper, &summary.DeveloperCount}, + {common.RoleGuest, &summary.GuestCount}, + } { + wg.Add(1) + go func(role int, count *int64) { + defer wg.Done() + + total, err := project.GetTotalOfProjectMembers(projectID, role) + if err != nil { + log.Debugf("failed to get total of project members of role %d", role) + return + } + + *count = total + }(e.role, e.count) + } + + wg.Wait() +} diff --git a/src/core/api/project_test.go b/src/core/api/project_test.go index 0db111288..2ff65b2fa 100644 --- a/src/core/api/project_test.go +++ b/src/core/api/project_test.go @@ -30,6 +30,42 @@ import ( var addProject *apilib.ProjectReq var addPID int +func addProjectByName(apiTest *testapi, projectName string) (int32, error) { + req := apilib.ProjectReq{ProjectName: projectName} + code, err := apiTest.ProjectsPost(*admin, req) + if err != nil { + return 0, err + } + if code != http.StatusCreated { + return 0, fmt.Errorf("created failed") + } + + code, projects, err := apiTest.ProjectsGet(&apilib.ProjectQuery{Name: projectName}, *admin) + if err != nil { + return 0, err + } + if code != http.StatusOK { + return 0, fmt.Errorf("get failed") + } + + if len(projects) == 0 { + return 0, fmt.Errorf("oops") + } + + return projects[0].ProjectId, nil +} + +func deleteProjectByIDs(apiTest *testapi, projectIDs ...int32) error { + for _, projectID := range projectIDs { + _, err := apiTest.ProjectsDelete(*admin, fmt.Sprintf("%d", projectID)) + if err != nil { + return err + } + } + + return nil +} + func InitAddPro() { addProject = &apilib.ProjectReq{ProjectName: "add_project", Metadata: map[string]string{models.ProMetaPublic: "true"}} } @@ -448,3 +484,30 @@ func TestDeletable(t *testing.T) { assert.Equal(t, http.StatusOK, code) assert.False(t, del) } + +func TestProjectSummary(t *testing.T) { + fmt.Println("\nTest for Project Summary API") + assert := assert.New(t) + + apiTest := newHarborAPI() + + projectID, err := addProjectByName(apiTest, "project-summary") + assert.Nil(err) + defer func() { + deleteProjectByIDs(apiTest, projectID) + }() + + // ----------------------------case 1 : Response Code=200----------------------------// + fmt.Println("case 1: respose code:200") + httpStatusCode, summary, err := apiTest.ProjectSummary(*admin, fmt.Sprintf("%d", projectID)) + if err != nil { + t.Error("Error while search project by proName", err.Error()) + t.Log(err) + } else { + assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") + assert.Equal(int64(1), summary.ProjectAdminCount) + assert.Equal(map[string]int64{"count": -1, "storage": -1}, summary.Quota.Hard) + } + + fmt.Printf("\n") +} diff --git a/src/core/api/quota.go b/src/core/api/quota.go new file mode 100644 index 000000000..eb55a6df3 --- /dev/null +++ b/src/core/api/quota.go @@ -0,0 +1,155 @@ +// Copyright 2018 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota" + "github.com/pkg/errors" +) + +// QuotaAPI handles request to /api/quotas/ +type QuotaAPI struct { + BaseController + quota *models.Quota +} + +// Prepare validates the URL and the user +func (qa *QuotaAPI) Prepare() { + qa.BaseController.Prepare() + + if !qa.SecurityCtx.IsAuthenticated() { + qa.SendUnAuthorizedError(errors.New("Unauthorized")) + return + } + + if !qa.SecurityCtx.IsSysAdmin() { + qa.SendForbiddenError(errors.New(qa.SecurityCtx.GetUsername())) + return + } + + if len(qa.GetStringFromPath(":id")) != 0 { + id, err := qa.GetInt64FromPath(":id") + if err != nil || id <= 0 { + text := "invalid quota ID: " + if err != nil { + text += err.Error() + } else { + text += fmt.Sprintf("%d", id) + } + qa.SendBadRequestError(errors.New(text)) + return + } + + quota, err := dao.GetQuota(id) + if err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to get quota %d, error: %v", id, err)) + return + } + + if quota == nil { + qa.SendNotFoundError(fmt.Errorf("quota %d not found", id)) + return + } + + qa.quota = quota + } +} + +// Get returns quota by id +func (qa *QuotaAPI) Get() { + query := &models.QuotaQuery{ + ID: qa.quota.ID, + } + + quotas, err := dao.ListQuotas(query) + if err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to get quota %d, error: %v", qa.quota.ID, err)) + return + } + + if len(quotas) == 0 { + qa.SendNotFoundError(fmt.Errorf("quota %d not found", qa.quota.ID)) + return + } + + qa.Data["json"] = quotas[0] + qa.ServeJSON() +} + +// Put update the quota +func (qa *QuotaAPI) Put() { + var req *models.QuotaUpdateRequest + if err := qa.DecodeJSONReq(&req); err != nil { + qa.SendBadRequestError(err) + return + } + + if err := quota.Validate(qa.quota.Reference, req.Hard); err != nil { + qa.SendBadRequestError(err) + return + } + + mgr, err := quota.NewManager(qa.quota.Reference, qa.quota.ReferenceID) + if err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to create quota manager, error: %v", err)) + return + } + + if err := mgr.UpdateQuota(req.Hard); err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to update hard limits of the quota, error: %v", err)) + return + } +} + +// List returns quotas by query +func (qa *QuotaAPI) List() { + page, size, err := qa.GetPaginationParams() + if err != nil { + qa.SendBadRequestError(err) + return + } + + query := &models.QuotaQuery{ + Reference: qa.GetString("reference"), + ReferenceID: qa.GetString("reference_id"), + Pagination: models.Pagination{ + Page: page, + Size: size, + }, + Sorting: models.Sorting{ + Sort: qa.GetString("sort"), + }, + } + + total, err := dao.GetTotalOfQuotas(query) + if err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to query database for total of quotas, error: %v", err)) + return + } + + quotas, err := dao.ListQuotas(query) + if err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to query database for quotas, error: %v", err)) + return + } + + qa.SetPaginationHeader(total, page, size) + qa.Data["json"] = quotas + qa.ServeJSON() +} diff --git a/src/core/api/quota_test.go b/src/core/api/quota_test.go new file mode 100644 index 000000000..ddda51457 --- /dev/null +++ b/src/core/api/quota_test.go @@ -0,0 +1,133 @@ +// Copyright 2018 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + "testing" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota" + "github.com/goharbor/harbor/src/common/quota/driver" + "github.com/goharbor/harbor/src/common/quota/driver/mocks" + "github.com/goharbor/harbor/src/pkg/types" + "github.com/goharbor/harbor/src/testing/apitests/apilib" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var ( + reference = "mock" + hardLimits = types.ResourceList{types.ResourceCount: -1, types.ResourceStorage: -1} +) + +func init() { + mockDriver := &mocks.Driver{} + + mockHardLimitsFn := func() types.ResourceList { + return hardLimits + } + + mockLoadFn := func(key string) driver.RefObject { + return driver.RefObject{"id": key} + } + + mockValidateFn := func(hardLimits types.ResourceList) error { + if len(hardLimits) == 0 { + return fmt.Errorf("no resources found") + } + + return nil + } + + mockDriver.On("HardLimits").Return(mockHardLimitsFn) + mockDriver.On("Load", mock.AnythingOfType("string")).Return(mockLoadFn, nil) + mockDriver.On("Validate", mock.AnythingOfType("types.ResourceList")).Return(mockValidateFn) + + driver.Register(reference, mockDriver) +} + +func TestQuotaAPIList(t *testing.T) { + assert := assert.New(t) + apiTest := newHarborAPI() + + count := 10 + for i := 0; i < count; i++ { + mgr, err := quota.NewManager(reference, fmt.Sprintf("%d", i)) + assert.Nil(err) + + _, err = mgr.NewQuota(hardLimits) + assert.Nil(err) + } + + code, quotas, err := apiTest.QuotasGet(&apilib.QuotaQuery{Reference: reference}, *admin) + assert.Nil(err) + assert.Equal(int(200), code) + assert.Len(quotas, count, fmt.Sprintf("quotas len should be %d", count)) + + code, quotas, err = apiTest.QuotasGet(&apilib.QuotaQuery{Reference: reference, PageSize: 1}, *admin) + assert.Nil(err) + assert.Equal(int(200), code) + assert.Len(quotas, 1) +} + +func TestQuotaAPIGet(t *testing.T) { + assert := assert.New(t) + apiTest := newHarborAPI() + + mgr, err := quota.NewManager(reference, "quota-get") + assert.Nil(err) + + quotaID, err := mgr.NewQuota(hardLimits) + assert.Nil(err) + + code, quota, err := apiTest.QuotasGetByID(*admin, fmt.Sprintf("%d", quotaID)) + assert.Nil(err) + assert.Equal(int(200), code) + assert.Equal(map[string]int64{"storage": -1, "count": -1}, quota.Hard) + + code, _, err = apiTest.QuotasGetByID(*admin, "100") + assert.Nil(err) + assert.Equal(int(404), code) +} + +func TestQuotaPut(t *testing.T) { + assert := assert.New(t) + apiTest := newHarborAPI() + + mgr, err := quota.NewManager(reference, "quota-put") + assert.Nil(err) + + quotaID, err := mgr.NewQuota(hardLimits) + assert.Nil(err) + + code, quota, err := apiTest.QuotasGetByID(*admin, fmt.Sprintf("%d", quotaID)) + assert.Nil(err) + assert.Equal(int(200), code) + assert.Equal(map[string]int64{"count": -1, "storage": -1}, quota.Hard) + + code, err = apiTest.QuotasPut(*admin, fmt.Sprintf("%d", quotaID), models.QuotaUpdateRequest{}) + assert.Nil(err, err) + assert.Equal(int(400), code) + + code, err = apiTest.QuotasPut(*admin, fmt.Sprintf("%d", quotaID), models.QuotaUpdateRequest{Hard: types.ResourceList{types.ResourceCount: 100, types.ResourceStorage: 100}}) + assert.Nil(err) + assert.Equal(int(200), code) + + code, quota, err = apiTest.QuotasGetByID(*admin, fmt.Sprintf("%d", quotaID)) + assert.Nil(err) + assert.Equal(int(200), code) + assert.Equal(map[string]int64{"count": 100, "storage": 100}, quota.Hard) +} diff --git a/src/core/router.go b/src/core/router.go index 5a857ee6f..15e178b9f 100644 --- a/src/core/router.go +++ b/src/core/router.go @@ -67,6 +67,7 @@ func initRouters() { beego.Router("/api/ping", &api.SystemInfoAPI{}, "get:Ping") beego.Router("/api/search", &api.SearchAPI{}) beego.Router("/api/projects/", &api.ProjectAPI{}, "get:List;post:Post") + beego.Router("/api/projects/:id([0-9]+)/summary", &api.ProjectAPI{}, "get:Summary") beego.Router("/api/projects/:id([0-9]+)/logs", &api.ProjectAPI{}, "get:Logs") beego.Router("/api/projects/:id([0-9]+)/_deletable", &api.ProjectAPI{}, "get:Deletable") beego.Router("/api/projects/:id([0-9]+)/metadatas/?:name", &api.MetadataAPI{}, "get:Get") @@ -76,6 +77,9 @@ func initRouters() { beego.Router("/api/projects/:pid([0-9]+)/robots", &api.RobotAPI{}, "post:Post;get:List") beego.Router("/api/projects/:pid([0-9]+)/robots/:id([0-9]+)", &api.RobotAPI{}, "get:Get;put:Put;delete:Delete") + beego.Router("/api/quotas", &api.QuotaAPI{}, "get:List") + beego.Router("/api/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put") + beego.Router("/api/repositories", &api.RepositoryAPI{}, "get:Get") beego.Router("/api/repositories/*", &api.RepositoryAPI{}, "delete:Delete;put:Put") beego.Router("/api/repositories/*/labels", &api.RepositoryLabelAPI{}, "get:GetOfRepository;post:AddToRepository") diff --git a/src/testing/apitests/apilib/project.go b/src/testing/apitests/apilib/project.go index d4d7cd398..13dcdfaf7 100644 --- a/src/testing/apitests/apilib/project.go +++ b/src/testing/apitests/apilib/project.go @@ -67,3 +67,19 @@ type ProjectQuery struct { Page int64 `url:"page,omitempty"` PageSize int64 `url:"page_size,omitempty"` } + +// ProjectSummary ... +type ProjectSummary struct { + RepoCount int64 `json:"repo_count"` + ChartCount uint64 `json:"chart_count"` + + ProjectAdminCount int64 `json:"project_admin_count"` + MasterCount int64 `json:"master_count"` + DeveloperCount int64 `json:"developer_count"` + GuestCount int64 `json:"guest_count"` + + Quota struct { + Hard map[string]int64 `json:"hard"` + Used map[string]int64 `json:"used"` + } `json:"quota"` +} diff --git a/src/testing/apitests/apilib/quota.go b/src/testing/apitests/apilib/quota.go new file mode 100644 index 000000000..288fb7918 --- /dev/null +++ b/src/testing/apitests/apilib/quota.go @@ -0,0 +1,39 @@ +/* + * Harbor API + * + * These APIs provide services for manipulating Harbor project. + * + * OpenAPI spec version: 0.3.0 + * + * Generated by: https://github.com/swagger-api/swagger-codegen.git + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package apilib + +// QuotaQuery query for quota +type QuotaQuery struct { + Reference string `url:"reference,omitempty"` + ReferenceID string `url:"reference_id,omitempty"` + Page int64 `url:"page,omitempty"` + PageSize int64 `url:"page_size,omitempty"` +} + +// Quota ... +type Quota struct { + ID int `json:"id"` + Ref map[string]interface{} `json:"ref"` + Hard map[string]int64 `json:"hard"` + Used map[string]int64 `json:"used"` +}