diff --git a/Makefile b/Makefile index e47f42b06..43a077e72 100644 --- a/Makefile +++ b/Makefile @@ -310,7 +310,7 @@ endif SWAGGER_IMAGENAME=goharbor/swagger SWAGGER_VERSION=v0.21.0 SWAGGER=$(DOCKERCMD) run --rm -u $(shell id -u):$(shell id -g) -v $(BUILDPATH):$(BUILDPATH) -w $(BUILDPATH) ${SWAGGER_IMAGENAME}:${SWAGGER_VERSION} -SWAGGER_GENERATE_SERVER=${SWAGGER} generate server --template-dir=$(TOOLSPATH)/swagger/templates --exclude-main +SWAGGER_GENERATE_SERVER=${SWAGGER} generate server --template-dir=$(TOOLSPATH)/swagger/templates --exclude-main --additional-initialism=CVE SWAGGER_IMAGE_BUILD_CMD=${DOCKERBUILD} -f ${TOOLSPATH}/swagger/Dockerfile --build-arg SWAGGER_VERSION=${SWAGGER_VERSION} -t ${SWAGGER_IMAGENAME}:$(SWAGGER_VERSION) . SWAGGER_IMAGENAME: diff --git a/api/v2.0/legacy_swagger.yaml b/api/v2.0/legacy_swagger.yaml index 16040c8ba..a27619fb0 100644 --- a/api/v2.0/legacy_swagger.yaml +++ b/api/v2.0/legacy_swagger.yaml @@ -53,214 +53,6 @@ paths: $ref: '#/definitions/Search' '500': description: Unexpected internal errors. - /projects: - get: - summary: List projects - description: | - This endpoint returns all projects created by Harbor, and can be filtered by project name. - parameters: - - name: name - in: query - description: The name of project. - required: false - type: string - - name: public - in: query - description: The project is public or private. - required: false - type: boolean - format: int32 - - name: owner - in: query - description: The name of project owner. - required: false - type: string - - name: page - in: query - type: integer - format: int32 - required: false - description: 'The page number, default is 1.' - - name: page_size - in: query - type: integer - format: int32 - required: false - description: 'The size of per page, default is 10, maximum is 100.' - tags: - - Products - responses: - '200': - description: Return all matched projects. - schema: - type: array - items: - $ref: '#/definitions/Project' - headers: - X-Total-Count: - description: The total count of projects - type: integer - Link: - description: Link refers to the previous page and next page - type: string - '401': - description: User need to log in first. - '500': - description: Internal errors. - head: - summary: Check if the project name user provided already exists. - description: | - This endpoint is used to check if the project name user provided already exist. - parameters: - - name: project_name - in: query - description: Project name for checking exists. - required: true - type: string - tags: - - Products - responses: - '200': - description: Project name exists. - '404': - description: Project name does not exist. - '500': - description: Unexpected internal errors. - post: - summary: Create a new project. - description: | - This endpoint is for user to create a new project. - parameters: - - name: project - in: body - description: New created project. - required: true - schema: - $ref: '#/definitions/ProjectReq' - tags: - - Products - responses: - '201': - description: Project created successfully. - '400': - description: Unsatisfied with constraints of the project creation. - '401': - description: User need to log in first. - '409': - description: Project name already exists. - '415': - $ref: '#/responses/UnsupportedMediaType' - '500': - description: Unexpected internal errors. - '/projects/{project_id}': - get: - summary: Return specific project detail information - description: | - This endpoint returns specific project information by project ID. - parameters: - - name: project_id - in: path - description: Project ID for filtering results. - required: true - type: integer - format: int64 - tags: - - Products - responses: - '200': - description: Return matched project information. - schema: - $ref: '#/definitions/Project' - '401': - description: User need to log in first. - '500': - description: Internal errors. - put: - summary: Update properties for a selected project. - description: | - This endpoint is aimed to update the properties of a project. - parameters: - - name: project_id - in: path - type: integer - format: int64 - required: true - description: Selected project ID. - - name: project - in: body - required: true - schema: - $ref: '#/definitions/ProjectReq' - description: Updates of project. - tags: - - Products - responses: - '200': - description: Updated project properties successfully. - '400': - description: Illegal format of provided ID value. - '401': - description: User need to log in first. - '403': - description: User does not have permission to the project. - '404': - description: Project ID does not exist. - '500': - description: Unexpected internal errors. - delete: - summary: Delete project by projectID - description: | - This endpoint is aimed to delete project by project ID. - parameters: - - name: project_id - in: path - description: Project ID of project which will be deleted. - required: true - type: integer - format: int64 - tags: - - Products - responses: - '200': - description: Project is deleted successfully. - '400': - description: Invalid project id. - '403': - description: User need to log in first. - '404': - description: Project does not exist. - '412': - description: 'Project contains policies, can not be deleted.' - '500': - description: Internal errors. - '/projects/{project_id}/summary': - get: - summary: Get summary of the project. - description: Get summary of the project. - parameters: - - name: project_id - in: path - type: integer - format: int64 - required: true - description: Relevant project ID - tags: - - Products - responses: - '200': - description: Get summary of the project successfully. - schema: - $ref: '#/definitions/ProjectSummary' - '400': - description: Illegal format of provided ID value. - '401': - description: User need to log in first. - '403': - description: User does not have permission to get summary of the project. - '404': - description: Project ID does not exist. - '500': - description: Unexpected internal errors. '/projects/{project_id}/metadatas': get: summary: Get project metadata. diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index ea5e1a307..8dfd3df76 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -20,6 +20,213 @@ security: - basic: [] - {} paths: + /projects: + get: + summary: List projects + description: This endpoint returns projects created by Harbor. + tags: + - project + operationId: listProjects + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + - name: name + in: query + description: The name of project. + required: false + type: string + - name: public + in: query + description: The project is public or private. + required: false + type: boolean + format: int32 + - name: owner + in: query + description: The name of project owner. + required: false + type: string + responses: + '200': + description: Return all matched projects. + schema: + type: array + items: + $ref: '#/definitions/Project' + headers: + X-Total-Count: + description: The total count of projects + type: integer + Link: + description: Link refers to the previous page and next page + type: string + '401': + $ref: '#/responses/401' + '500': + $ref: '#/responses/500' + head: + summary: Check if the project name user provided already exists. + description: This endpoint is used to check if the project name provided already exist. + tags: + - project + operationId: headProject + parameters: + - $ref: '#/parameters/requestId' + - name: project_name + in: query + description: Project name for checking exists. + required: true + type: string + responses: + '200': + $ref: '#/responses/200' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + post: + summary: Create a new project. + description: This endpoint is for user to create a new project. + tags: + - project + operationId: createProject + parameters: + - $ref: '#/parameters/requestId' + - name: project + in: body + description: New created project. + required: true + schema: + $ref: '#/definitions/ProjectReq' + responses: + '201': + $ref: '#/responses/201' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '409': + $ref: '#/responses/409' + '500': + $ref: '#/responses/500' + '/projects/{project_id}': + get: + summary: Return specific project detail information + description: This endpoint returns specific project information by project ID. + tags: + - project + operationId: getProject + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/projectId' + responses: + '200': + description: Return matched project information. + schema: + $ref: '#/definitions/Project' + '401': + $ref: '#/responses/401' + '500': + $ref: '#/responses/500' + put: + summary: Update properties for a selected project. + description: This endpoint is aimed to update the properties of a project. + tags: + - project + operationId: updateProject + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/projectId' + - name: project + in: body + required: true + schema: + $ref: '#/definitions/ProjectReq' + description: Updates of project. + responses: + '200': + $ref: '#/responses/200' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + delete: + summary: Delete project by projectID + description: This endpoint is aimed to delete project by project ID. + tags: + - project + operationId: deleteProject + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/projectId' + responses: + '200': + $ref: '#/responses/200' + '400': + $ref: '#/responses/400' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '412': + $ref: '#/responses/412' + '500': + $ref: '#/responses/500' + /projects/{project_id}/_deletable: + get: + summary: Get the deletable status of the project + description: Get the deletable status of the project + tags: + - project + operationId: getProjectDeletable + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/projectId' + responses: + '200': + description: Return deletable status of the project. + schema: + $ref: '#/definitions/ProjectDeletable' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + '/projects/{project_id}/summary': + get: + summary: Get summary of the project. + description: Get summary of the project. + tags: + - project + operationId: getProjectSummary + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/projectId' + responses: + '200': + description: Get summary of the project successfully. + schema: + $ref: '#/definitions/ProjectSummary' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' /projects/{project_name}/repositories: get: summary: List repositories @@ -1252,7 +1459,8 @@ parameters: in: path description: The ID of the project required: true - type: string + type: integer + format: int64 repositoryName: name: repository_name in: path @@ -1293,6 +1501,7 @@ parameters: required: false description: The size of per page default: 10 + maximum: 100 instanceName: name: preheat_instance_name in: path @@ -1387,6 +1596,14 @@ responses: type: string schema: $ref: '#/definitions/Errors' + '412': + description: Precondition failed + headers: + X-Request-Id: + description: The ID of the corresponding request for the response + type: string + schema: + $ref: '#/definitions/Errors' '500': description: Internal server error headers: @@ -1958,3 +2175,243 @@ definitions: content: type: string description: The base64 encoded content of the icon + ProjectReq: + type: object + properties: + project_name: + type: string + description: The name of the project. + public: + type: boolean + description: deprecated, reserved for project creation in replication + x-nullable: true + metadata: + description: The metadata of the project. + $ref: '#/definitions/ProjectMetadata' + cve_allowlist: + description: The CVE allowlist of the project. + $ref: '#/definitions/CVEAllowlist' + storage_limit: + type: integer + format: int64 + description: The storage quota of the project. + x-nullable: true + registry_id: + type: integer + format: int64 + description: The ID of referenced registry when creating the proxy cache project + x-nullable: true + Project: + type: object + properties: + project_id: + type: integer + format: int32 + description: Project ID + owner_id: + type: integer + format: int32 + description: The owner ID of the project always means the creator of the project. + name: + type: string + description: The name of the project. + registry_id: + type: integer + format: int64 + description: The ID of referenced registry when the project is a proxy cache project. + creation_time: + type: string + format: date-time + description: The creation time of the project. + update_time: + type: string + format: date-time + description: The update time of the project. + deleted: + type: boolean + description: A deletion mark of the project. + owner_name: + type: string + description: The owner name of the project. + togglable: + type: boolean + description: Correspond to the UI about whether the project's publicity is updatable (for UI) + current_user_role_id: + type: integer + description: The role ID with highest permission of the current user who triggered the API (for UI). This attribute is deprecated and will be removed in future versions. + current_user_role_ids: + type: array + items: + type: integer + format: int32 + description: The list of role ID of the current user who triggered the API (for UI) + repo_count: + type: integer + description: The number of the repositories under this project. + chart_count: + type: integer + description: The total number of charts under this project. + metadata: + description: The metadata of the project. + $ref: '#/definitions/ProjectMetadata' + cve_allowlist: + description: The CVE allowlist of this project. + $ref: '#/definitions/CVEAllowlist' + ProjectDeletable: + type: object + properties: + deletable: + type: boolean + description: Whether the project can be deleted. + message: + type: string + description: The detail message when the project can not be deleted. + ProjectMetadata: + type: object + properties: + public: + type: string + description: 'The public status of the project. The valid values are "true", "false".' + enable_content_trust: + type: string + description: 'Whether content trust is enabled or not. If it is enabled, user can''t pull unsigned images from this project. The valid values are "true", "false".' + x-nullable: true + prevent_vul: + type: string + description: 'Whether prevent the vulnerable images from running. The valid values are "true", "false".' + x-nullable: true + severity: + type: string + description: 'If the vulnerability is high than severity defined here, the images can''t be pulled. The valid values are "none", "low", "medium", "high", "critical".' + x-nullable: true + auto_scan: + type: string + description: 'Whether scan images automatically when pushing. The valid values are "true", "false".' + x-nullable: true + reuse_sys_cve_allowlist: + type: string + description: 'Whether this project reuse the system level CVE allowlist as the allowlist of its own. The valid values are "true", "false". + If it is set to "true" the actual allowlist associate with this project, if any, will be ignored.' + x-nullable: true + retention_id: + type: string + description: 'The ID of the tag retention policy for the project' + x-nullable: true + ProjectSummary: + type: object + properties: + repo_count: + type: integer + description: The number of the repositories under this project. + chart_count: + type: integer + description: The total number of charts under this project. + project_admin_count: + type: integer + description: The total number of project admin members. + maintainer_count: + type: integer + description: The total number of maintainer members. + developer_count: + type: integer + description: The total number of developer members. + guest_count: + type: integer + description: The total number of guest members. + limited_guest_count: + type: integer + description: The total number of limited guest members. + quota: + type: object + properties: + hard: + $ref: "#/definitions/ResourceList" + description: The hard limits of the quota + used: + $ref: "#/definitions/ResourceList" + description: The used status of the quota + registry: + $ref: "#/definitions/Registry" + CVEAllowlist: + type: object + description: The CVE Allowlist for system or project + properties: + id: + type: integer + description: ID of the allowlist + project_id: + type: integer + description: ID of the project which the allowlist belongs to. For system level allowlist this attribute is zero. + expires_at: + type: integer + description: the time for expiration of the allowlist, in the form of seconds since epoch. This is an optional attribute, if it's not set the CVE allowlist does not expire. + x-nullable: true + items: + type: array + items: + $ref: "#/definitions/CVEAllowlistItem" + creation_time: + type: string + format: date-time + description: The creation time of the allowlist. + update_time: + type: string + format: date-time + description: The update time of the allowlist. + CVEAllowlistItem: + type: object + description: The item in CVE allowlist + properties: + cve_id: + type: string + description: The ID of the CVE, such as "CVE-2019-10164" + RegistryCredential: + type: object + properties: + type: + type: string + description: Credential type, such as 'basic', 'oauth'. + access_key: + type: string + description: Access key, e.g. user name when credential type is 'basic'. + access_secret: + type: string + description: Access secret, e.g. password when credential type is 'basic'. + Registry: + type: object + properties: + id: + type: integer + format: int64 + description: The registry ID. + url: + type: string + description: The registry URL string. + name: + type: string + description: The registry name. + credential: + $ref: '#/definitions/RegistryCredential' + type: + type: string + description: Type of the registry, e.g. 'harbor'. + insecure: + type: boolean + description: Whether or not the certificate will be verified when Harbor tries to access the server. + description: + type: string + description: Description of the registry. + status: + type: string + description: Health status of the registry. + creation_time: + type: string + description: The create time of the policy. + update_time: + type: string + description: The update time of the policy. + ResourceList: + type: object + additionalProperties: + type: integer + format: int64 diff --git a/src/common/dao/cve_allowlist.go b/src/common/dao/cve_allowlist.go index 68a6ee207..c8ed55fc1 100644 --- a/src/common/dao/cve_allowlist.go +++ b/src/common/dao/cve_allowlist.go @@ -17,6 +17,8 @@ package dao import ( "encoding/json" "fmt" + "time" + "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/lib/log" ) @@ -24,6 +26,9 @@ import ( // CreateCVEAllowlist creates the CVE allowlist func CreateCVEAllowlist(l models.CVEAllowlist) (int64, error) { o := GetOrmer() + now := time.Now() + l.CreationTime = now + l.UpdateTime = now itemsBytes, _ := json.Marshal(l.Items) l.ItemsText = string(itemsBytes) return o.Insert(&l) @@ -32,6 +37,8 @@ func CreateCVEAllowlist(l models.CVEAllowlist) (int64, error) { // UpdateCVEAllowlist Updates the vulnerability white list to DB func UpdateCVEAllowlist(l models.CVEAllowlist) (int64, error) { o := GetOrmer() + now := time.Now() + l.UpdateTime = now itemsBytes, _ := json.Marshal(l.Items) l.ItemsText = string(itemsBytes) id, err := o.InsertOrUpdate(&l, "project_id") diff --git a/src/common/models/project.go b/src/common/models/project.go index daef9033e..c89b0aba5 100644 --- a/src/common/models/project.go +++ b/src/common/models/project.go @@ -22,7 +22,6 @@ import ( "time" "github.com/astaxie/beego/orm" - "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/quota/types" "github.com/goharbor/harbor/src/replication/model" ) @@ -154,6 +153,10 @@ func (p *Project) FilterByMember(ctx context.Context, qs orm.QuerySeter, key str subQuery = fmt.Sprintf("%s AND pm.role = %d", subQuery, query.Role) } + if query.WithPublic { + subQuery = fmt.Sprintf("(%s) UNION (SELECT project_id FROM project_metadata WHERE name = 'public' AND value = 'true')", subQuery) + } + if len(query.GroupIDs) > 0 { var elems []string for _, groupID := range query.GroupIDs { @@ -210,41 +213,13 @@ type ProjectQueryParam struct { ProjectIDs []int64 // project ID list } -// ToQuery returns q.Query from param -func (param *ProjectQueryParam) ToQuery() *q.Query { - kw := q.KeyWords{} - if param.Name != "" { - kw["name"] = q.FuzzyMatchValue{Value: param.Name} - } - if param.Owner != "" { - kw["owner"] = param.Owner - } - if param.Public != nil { - kw["public"] = *param.Public - } - if param.RegistryID != 0 { - kw["registry_id"] = param.RegistryID - } - if len(param.ProjectIDs) > 0 { - kw["project_id__in"] = param.ProjectIDs - } - if param.Member != nil { - kw["member"] = param.Member - } - - query := q.New(kw) - if param.Pagination != nil { - query.PageNumber = param.Pagination.Page - query.PageSize = param.Pagination.Size - } - return query -} - // MemberQuery filter by member's username and role type MemberQuery struct { Name string // the username of member Role int // the role of the member has to the project GroupIDs []int // the group ID of current user belongs to + + WithPublic bool // include the public projects for the member } // Pagination ... diff --git a/src/controller/event/operator/operator.go b/src/controller/event/operator/operator.go new file mode 100644 index 000000000..9bfc455bd --- /dev/null +++ b/src/controller/event/operator/operator.go @@ -0,0 +1,41 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + "context" + + "github.com/goharbor/harbor/src/common/security" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/user" +) + +// FromContext return the event operator from context +func FromContext(ctx context.Context) string { + sc, ok := security.FromContext(ctx) + if !ok { + return "" + } + + if sc.IsSolutionUser() { + user, err := user.Mgr.Get(ctx, 1) + if err == nil { + return user.Username + } + log.G(ctx).Errorf("failed to get operator for security %s, error: %v", sc.Name(), err) + } + + return sc.GetUsername() +} diff --git a/src/controller/p2p/preheat/enforcer.go b/src/controller/p2p/preheat/enforcer.go index 4bac893e5..c9fd55dd3 100644 --- a/src/controller/p2p/preheat/enforcer.go +++ b/src/controller/p2p/preheat/enforcer.go @@ -539,7 +539,7 @@ func (de *defaultEnforcer) toCandidates(ctx context.Context, p *models.Project, // getProject gets the full metadata of the specified project func (de *defaultEnforcer) getProject(ctx context.Context, id int64) (*models.Project, error) { // Get project info with CVE allow list and metadata - return de.proCtl.Get(ctx, id, project.CVEAllowlist(true), project.Metadata(true)) + return de.proCtl.Get(ctx, id, project.WithEffectCVEAllowlist()) } // enforceError is a wrap error diff --git a/src/controller/project/controller.go b/src/controller/project/controller.go index 8997b1c9d..1f95704c3 100644 --- a/src/controller/project/controller.go +++ b/src/controller/project/controller.go @@ -17,10 +17,13 @@ package project import ( "context" + event "github.com/goharbor/harbor/src/controller/event/metadata" + "github.com/goharbor/harbor/src/controller/event/operator" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/project" "github.com/goharbor/harbor/src/pkg/project/metadata" "github.com/goharbor/harbor/src/pkg/project/models" @@ -33,6 +36,12 @@ var ( Ctl = NewController() ) +// Project alias to models.Project +type Project = models.Project + +// MemberQuery alias to models.MemberQuery +type MemberQuery = models.MemberQuery + // Controller defines the operations related with blobs type Controller interface { // Create create project instance @@ -46,7 +55,9 @@ type Controller interface { // GetByName get the project by project name GetByName(ctx context.Context, projectName string, options ...Option) (*models.Project, error) // List list projects - List(ctx context.Context, query *models.ProjectQueryParam, options ...Option) ([]*models.Project, error) + List(ctx context.Context, query *q.Query, options ...Option) ([]*models.Project, error) + // Update update the project + Update(ctx context.Context, project *models.Project) error } // NewController creates an instance of the default project controller @@ -92,6 +103,14 @@ func (c *controller) Create(ctx context.Context, project *models.Project) (int64 return 0, err } + // fire event + e := &event.CreateProjectEventMetadata{ + ProjectID: projectID, + Project: project.Name, + Operator: operator.FromContext(ctx), + } + notification.AddEvent(ctx, e) + return projectID, nil } @@ -100,7 +119,23 @@ func (c *controller) Count(ctx context.Context, query *q.Query) (int64, error) { } func (c *controller) Delete(ctx context.Context, id int64) error { - return c.projectMgr.Delete(ctx, id) + proj, err := c.Get(ctx, id) + if err != nil { + return err + } + + if err := c.projectMgr.Delete(ctx, id); err != nil { + return err + } + + e := &event.DeleteProjectEventMetadata{ + ProjectID: proj.ProjectID, + Project: proj.Name, + Operator: operator.FromContext(ctx), + } + notification.AddEvent(ctx, e) + + return nil } func (c *controller) Get(ctx context.Context, projectID int64, options ...Option) (*models.Project, error) { @@ -109,13 +144,11 @@ func (c *controller) Get(ctx context.Context, projectID int64, options ...Option return nil, err } - opts := newOptions(options...) - if opts.WithOwner { - if err := c.loadOwners(ctx, models.Projects{p}); err != nil { - return nil, err - } + if err := c.assembleProjects(ctx, models.Projects{p}, options...); err != nil { + return nil, err } - return c.assembleProject(ctx, p, opts) + + return p, nil } func (c *controller) GetByName(ctx context.Context, projectName string, options ...Option) (*models.Project, error) { @@ -128,38 +161,164 @@ func (c *controller) GetByName(ctx context.Context, projectName string, options return nil, err } - opts := newOptions(options...) - if opts.WithOwner { - if err := c.loadOwners(ctx, models.Projects{p}); err != nil { - return nil, err - } + if err := c.assembleProjects(ctx, models.Projects{p}, options...); err != nil { + return nil, err } - return c.assembleProject(ctx, p, newOptions(options...)) + + return p, nil } -func (c *controller) List(ctx context.Context, query *models.ProjectQueryParam, options ...Option) ([]*models.Project, error) { +func (c *controller) List(ctx context.Context, query *q.Query, options ...Option) ([]*models.Project, error) { projects, err := c.projectMgr.List(ctx, query) if err != nil { return nil, err } - opts := newOptions(options...) - if opts.WithOwner { - if err := c.loadOwners(ctx, projects); err != nil { - return nil, err - } + if len(projects) == 0 { + return projects, nil } - for _, p := range projects { - if _, err := c.assembleProject(ctx, p, opts); err != nil { - return nil, err - } + if err := c.assembleProjects(ctx, projects, options...); err != nil { + return nil, err } return projects, nil } +func (c *controller) Update(ctx context.Context, p *models.Project) error { + // currently, allowlist manager not use the ormer from the context, + // the SQL executed in the allowlist manager will not be in the transaction with metadata manager, + // we will update the metadata of the project first so that we can be rollback the operations for the metadata + // when set allowlist for the project failed + if len(p.Metadata) > 0 { + meta, err := c.metaMgr.Get(ctx, p.ProjectID) + if err != nil { + return err + } + if meta == nil { + meta = map[string]string{} + } + + metaNeedUpdated := map[string]string{} + metaNeedCreated := map[string]string{} + for key, value := range p.Metadata { + _, exist := meta[key] + if exist { + metaNeedUpdated[key] = value + } else { + metaNeedCreated[key] = value + } + } + if err = c.metaMgr.Add(ctx, p.ProjectID, metaNeedCreated); err != nil { + return err + } + if err = c.metaMgr.Update(ctx, p.ProjectID, metaNeedUpdated); err != nil { + return err + } + } + + if p.CVEAllowlist.ProjectID == p.ProjectID { + if err := c.allowlistMgr.Set(p.ProjectID, p.CVEAllowlist); err != nil { + return err + } + } + + return nil +} + +func (c *controller) assembleProjects(ctx context.Context, projects models.Projects, options ...Option) error { + opts := newOptions(options...) + if opts.WithMetadata { + if err := c.loadMetadatas(ctx, projects); err != nil { + return err + } + } + + if opts.WithEffectCVEAllowlist { + if err := c.loadEffectCVEAllowlists(ctx, projects); err != nil { + return err + } + } else if opts.WithCVEAllowlist { + if err := c.loadCVEAllowlists(ctx, projects); err != nil { + return err + } + } + + if opts.WithOwner { + if err := c.loadOwners(ctx, projects); err != nil { + return err + } + } + + return nil +} + +func (c *controller) loadCVEAllowlists(ctx context.Context, projects models.Projects) error { + if len(projects) == 0 { + return nil + } + + for _, p := range projects { + wl, err := c.allowlistMgr.Get(p.ProjectID) + if err != nil { + return err + } + + p.CVEAllowlist = *wl + } + + return nil +} + +func (c *controller) loadEffectCVEAllowlists(ctx context.Context, projects models.Projects) error { + if len(projects) == 0 { + return nil + } + + for _, p := range projects { + if p.ReuseSysCVEAllowlist() { + wl, err := c.allowlistMgr.GetSys() + if err != nil { + log.Errorf("get system CVE allowlist failed, error: %v", err) + return err + } + + wl.ProjectID = p.ProjectID + p.CVEAllowlist = *wl + } else { + wl, err := c.allowlistMgr.Get(p.ProjectID) + if err != nil { + return err + } + + p.CVEAllowlist = *wl + } + } + + return nil +} + +func (c *controller) loadMetadatas(ctx context.Context, projects models.Projects) error { + if len(projects) == 0 { + return nil + } + + for _, p := range projects { + meta, err := c.metaMgr.Get(ctx, p.ProjectID) + if err != nil { + return err + } + p.Metadata = meta + } + + return nil +} + func (c *controller) loadOwners(ctx context.Context, projects models.Projects) error { + if len(projects) == 0 { + return nil + } + owners, err := c.userMgr.List(ctx, q.New(q.KeyWords{"user_id__in": projects.OwnerIDs()})) if err != nil { return err @@ -177,42 +336,3 @@ func (c *controller) loadOwners(ctx context.Context, projects models.Projects) e return nil } - -func (c *controller) assembleProject(ctx context.Context, p *models.Project, opts *Options) (*models.Project, error) { - if opts.Metadata { - meta, err := c.metaMgr.Get(ctx, p.ProjectID) - if err != nil { - return nil, err - } - if len(p.Metadata) == 0 { - p.Metadata = make(map[string]string) - } - - for k, v := range meta { - p.Metadata[k] = v - } - } - - if opts.CVEAllowlist { - if p.ReuseSysCVEAllowlist() { - wl, err := c.allowlistMgr.GetSys() - if err != nil { - log.Errorf("get system CVE allowlist failed, error: %v", err) - return nil, err - } - - wl.ProjectID = p.ProjectID - p.CVEAllowlist = *wl - } else { - wl, err := c.allowlistMgr.Get(p.ProjectID) - if err != nil { - return nil, err - } - - p.CVEAllowlist = *wl - } - - } - - return p, nil -} diff --git a/src/controller/project/controller_test.go b/src/controller/project/controller_test.go index 4415f9bf1..710cfebeb 100644 --- a/src/controller/project/controller_test.go +++ b/src/controller/project/controller_test.go @@ -22,6 +22,7 @@ import ( commonmodels "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/project/models" usermodels "github.com/goharbor/harbor/src/pkg/user/models" ormtesting "github.com/goharbor/harbor/src/testing/lib/orm" @@ -102,8 +103,8 @@ func (suite *ControllerTestSuite) TestGetByName() { } { - allowlistMgr.On("GetSys").Return(&commonmodels.CVEAllowlist{}, nil) - p, err := c.GetByName(ctx, "library", CVEAllowlist(true)) + allowlistMgr.On("Get", mock.Anything).Return(&commonmodels.CVEAllowlist{ProjectID: 1}, nil) + p, err := c.GetByName(ctx, "library", WithCVEAllowlist()) suite.Nil(err) suite.Equal("library", p.Name) suite.Equal(p.ProjectID, p.CVEAllowlist.ProjectID) @@ -140,8 +141,7 @@ func (suite *ControllerTestSuite) TestWithOwner() { } { - param := &models.ProjectQueryParam{ProjectIDs: []int64{1}} - projects, err := c.List(ctx, param, Metadata(false), WithOwner()) + projects, err := c.List(ctx, q.New(q.KeyWords{"project_id__in": []int64{1}}), Metadata(false), WithOwner()) suite.Nil(err) suite.Len(projects, 1) suite.Equal("admin", projects[0].OwnerName) diff --git a/src/controller/project/options.go b/src/controller/project/options.go index 11902f951..256b668c4 100644 --- a/src/controller/project/options.go +++ b/src/controller/project/options.go @@ -19,22 +19,31 @@ type Option func(*Options) // Options options used by `Get` method of `Controller` type Options struct { - CVEAllowlist bool // get project with cve allowlist - Metadata bool // get project with metadata - WithOwner bool + WithCVEAllowlist bool // get project with cve allowlist + WithEffectCVEAllowlist bool // get project with effect cve allowlist + WithMetadata bool // get project with metadata + WithOwner bool // get project with owner name } -// CVEAllowlist set CVEAllowlist for the Options -func CVEAllowlist(allowlist bool) Option { +// WithCVEAllowlist set WithCVEAllowlist for the Options +func WithCVEAllowlist() Option { return func(opts *Options) { - opts.CVEAllowlist = allowlist + opts.WithCVEAllowlist = true } } -// Metadata set Metadata for the Options +// WithEffectCVEAllowlist set WithEffectCVEAllowlist for the Options +func WithEffectCVEAllowlist() Option { + return func(opts *Options) { + opts.WithMetadata = true // we need `reuse_sys_cve_allowlist` value in the metadata of project + opts.WithEffectCVEAllowlist = true + } +} + +// Metadata set WithMetadata for the Options func Metadata(metadata bool) Option { return func(opts *Options) { - opts.Metadata = metadata + opts.WithMetadata = metadata } } @@ -47,7 +56,7 @@ func WithOwner() Option { func newOptions(options ...Option) *Options { opts := &Options{ - Metadata: true, // default get project with metadata + WithMetadata: true, // default get project with metadata } for _, f := range options { diff --git a/src/controller/project/util.go b/src/controller/project/util.go new file mode 100644 index 000000000..c1cd347ff --- /dev/null +++ b/src/controller/project/util.go @@ -0,0 +1,64 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package project + +import ( + "context" + "fmt" + + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/project/models" +) + +// Result the result for ListAll func +type Result struct { + Data *models.Project + Error error +} + +// ListAll returns all projects with chunk support +func ListAll(ctx context.Context, chunkSize int, query *q.Query, options ...Option) <-chan Result { + ch := make(chan Result, chunkSize) + + go func() { + defer close(ch) + + query = q.MustClone(query) + query.PageNumber = 1 + query.PageSize = int64(chunkSize) + + for { + projects, err := Ctl.List(ctx, query, options...) + if err != nil { + format := "failed to list projects at page %d with page size %d, error :%v" + ch <- Result{Error: fmt.Errorf(format, query.PageNumber, query.PageSize, err)} + return + } + + for _, p := range projects { + ch <- Result{Data: p} + } + + if len(projects) < chunkSize { + break + } + + query.PageNumber++ + } + + }() + + return ch +} diff --git a/src/controller/quota/driver/project/util.go b/src/controller/quota/driver/project/util.go index 1741508f1..1341cbaf3 100644 --- a/src/controller/quota/driver/project/util.go +++ b/src/controller/quota/driver/project/util.go @@ -21,6 +21,7 @@ import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/project" "github.com/graph-gophers/dataloader" ) @@ -43,7 +44,7 @@ func getProjectsBatchFn(ctx context.Context, keys dataloader.Keys) []*dataloader projectIDs = append(projectIDs, id) } - projects, err := project.Mgr.List(ctx, &models.ProjectQueryParam{ProjectIDs: projectIDs}) + projects, err := project.Mgr.List(ctx, q.New(q.KeyWords{"project_id__in": projectIDs})) if err != nil { return handleError(err) } diff --git a/src/controller/quota/util.go b/src/controller/quota/util.go index 6ddd22f4f..22c16f79e 100644 --- a/src/controller/quota/util.go +++ b/src/controller/quota/util.go @@ -19,7 +19,6 @@ import ( "fmt" "strconv" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" @@ -55,40 +54,14 @@ func RefreshForProjects(ctx context.Context) error { return err } - projects := func(chunkSize int) <-chan *models.Project { - ch := make(chan *models.Project, chunkSize) + chunkSize := 50 // default chunk size is 50 + for result := range project.ListAll(ctx, chunkSize, nil, project.Metadata(false)) { + if result.Error != nil { + log.Errorf("refresh quota for all projects got error: %v", result.Error) + continue + } - go func() { - defer close(ch) - - params := &models.ProjectQueryParam{ - Pagination: &models.Pagination{Page: 1, Size: int64(chunkSize)}, - } - - for { - results, err := project.Ctl.List(ctx, params, project.Metadata(false)) - if err != nil { - log.Errorf("list projects failed, error: %v", err) - return - } - - for _, p := range results { - ch <- p - } - - if len(results) < chunkSize { - break - } - - params.Pagination.Page++ - } - - }() - - return ch - }(50) // default chunk size is 50 - - for p := range projects { + p := result.Data referenceID := ReferenceID(p.ProjectID) _, err := Ctl.GetByRef(ctx, ProjectReference, referenceID) diff --git a/src/controller/quota/util_test.go b/src/controller/quota/util_test.go index 059cd4638..31c3ecf45 100644 --- a/src/controller/quota/util_test.go +++ b/src/controller/quota/util_test.go @@ -23,6 +23,7 @@ import ( "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/quota" "github.com/goharbor/harbor/src/pkg/quota/driver" "github.com/goharbor/harbor/src/pkg/quota/types" @@ -89,7 +90,7 @@ func (suite *RefreshForProjectsTestSuite) TestRefreshForProjects() { } page := 1 - mock.OnAnything(suite.projectCtl, "List").Return(func(context.Context, *models.ProjectQueryParam, ...project.Option) []*models.Project { + mock.OnAnything(suite.projectCtl, "List").Return(func(context.Context, *q.Query, ...project.Option) []*models.Project { defer func() { page++ }() diff --git a/src/core/api/base.go b/src/core/api/base.go index ca1ab05e7..3c22aaa38 100644 --- a/src/core/api/base.go +++ b/src/core/api/base.go @@ -51,6 +51,11 @@ var ( retentionController retention.APIController ) +// GetRetentionController returns the retention API controller +func GetRetentionController() retention.APIController { + return retentionController +} + // BaseController ... type BaseController struct { api.BaseAPI diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 9a26506f3..acd293636 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -97,16 +97,12 @@ func init() { beego.Router("/api/health", &HealthAPI{}, "get:CheckHealth") beego.Router("/api/search/", &SearchAPI{}) - beego.Router("/api/projects/", &ProjectAPI{}, "get:List;post:Post;head:Head") - beego.Router("/api/projects/:id", &ProjectAPI{}, "delete:Delete;get:Get;put:Put") beego.Router("/api/users/:id", &UserAPI{}, "get:Get") beego.Router("/api/users", &UserAPI{}, "get:List;post:Post;delete:Delete;put:Put") beego.Router("/api/users/search", &UserAPI{}, "get:Search") beego.Router("/api/users/:id([0-9]+)/password", &UserAPI{}, "put:ChangePassword") beego.Router("/api/users/:id/permissions", &UserAPI{}, "get:ListUserPermissions") beego.Router("/api/users/:id/sysadmin", &UserAPI{}, "put:ToggleUserAdminRole") - beego.Router("/api/projects/:id([0-9]+)/summary", &ProjectAPI{}, "get:Summary") - beego.Router("/api/projects/:id([0-9]+)/_deletable", &ProjectAPI{}, "get:Deletable") beego.Router("/api/projects/:id([0-9]+)/metadatas/?:name", &MetadataAPI{}, "get:Get") beego.Router("/api/projects/:id([0-9]+)/metadatas/", &MetadataAPI{}, "post:Post") beego.Router("/api/projects/:id([0-9]+)/metadatas/:name", &MetadataAPI{}, "put:Put;delete:Delete") diff --git a/src/core/api/project.go b/src/core/api/project.go deleted file mode 100644 index 86970cbbc..000000000 --- a/src/core/api/project.go +++ /dev/null @@ -1,720 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "context" - "fmt" - "net/http" - "regexp" - "strconv" - "strings" - "sync" - - "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/dao" - pro "github.com/goharbor/harbor/src/common/dao/project" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/rbac" - "github.com/goharbor/harbor/src/common/security/local" - "github.com/goharbor/harbor/src/common/utils" - errutil "github.com/goharbor/harbor/src/common/utils/error" - "github.com/goharbor/harbor/src/controller/event/metadata" - "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/controller/quota" - "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/lib/errors" - "github.com/goharbor/harbor/src/lib/log" - evt "github.com/goharbor/harbor/src/pkg/notifier/event" - "github.com/goharbor/harbor/src/pkg/quota/types" - "github.com/goharbor/harbor/src/pkg/retention/policy" - "github.com/goharbor/harbor/src/pkg/scan/vuln" - "github.com/goharbor/harbor/src/replication" -) - -type deletableResp struct { - Deletable bool `json:"deletable"` - Message string `json:"message"` -} - -// ProjectAPI handles request to /api/projects/{} /api/projects/{}/logs -type ProjectAPI struct { - BaseController - project *models.Project -} - -const projectNameMaxLen int = 255 -const projectNameMinLen int = 1 -const restrictedNameChars = `[a-z0-9]+(?:[._-][a-z0-9]+)*` -const defaultDaysToRetention = 7 - -// Prepare validates the URL and the user -func (p *ProjectAPI) Prepare() { - p.BaseController.Prepare() - if len(p.GetStringFromPath(":id")) != 0 { - id, err := p.GetInt64FromPath(":id") - if err != nil || id <= 0 { - text := "invalid project ID: " - if err != nil { - text += err.Error() - } else { - text += fmt.Sprintf("%d", id) - } - p.SendBadRequestError(errors.New(text)) - return - } - - project, err := p.ProjectMgr.Get(id) - if err != nil { - p.ParseAndHandleError(fmt.Sprintf("failed to get project %d", id), err) - return - } - - if project == nil { - p.handleProjectNotFound(id) - return - } - - p.project = project - } -} - -func (p *ProjectAPI) requireAccess(action rbac.Action, subresource ...rbac.Resource) bool { - if len(subresource) == 0 { - subresource = append(subresource, rbac.ResourceSelf) - } - - return p.RequireProjectAccess(p.project.ProjectID, action, subresource...) -} - -// Post ... -func (p *ProjectAPI) Post() { - if !p.SecurityCtx.IsAuthenticated() { - p.SendUnAuthorizedError(errors.New("Unauthorized")) - return - } - onlyAdmin, err := config.OnlyAdminCreateProject() - if err != nil { - log.Errorf("failed to determine whether only admin can create projects: %v", err) - p.SendInternalServerError(fmt.Errorf("failed to determine whether only admin can create projects: %v", err)) - return - } - - if onlyAdmin && !(p.SecurityCtx.IsSysAdmin() || p.SecurityCtx.IsSolutionUser()) { - log.Errorf("Only sys admin can create project") - p.SendForbiddenError(errors.New("Only system admin can create project")) - return - } - var pro *models.ProjectRequest - if err := p.DecodeJSONReq(&pro); err != nil { - p.SendBadRequestError(err) - return - } - - err = validateProjectReq(pro) - if err != nil { - log.Errorf("Invalid project request, error: %v", err) - p.SendBadRequestError(fmt.Errorf("invalid request: %v", err)) - return - } - - // trying to create a proxy cache project - if pro.RegistryID > 0 { - // only system admin can create the proxy cache project - if !p.SecurityCtx.IsSysAdmin() { - p.SendForbiddenError(errors.New("Only system admin can create proxy cache project")) - return - } - registry, err := replication.RegistryMgr.Get(pro.RegistryID) - if err != nil { - p.SendInternalServerError(fmt.Errorf("failed to get the registry %d: %v", pro.RegistryID, err)) - return - } - if registry == nil { - p.SendNotFoundError(fmt.Errorf("registry %d not found", pro.RegistryID)) - return - } - permitted := false - for _, t := range config.GetPermittedRegistryTypesForProxyCache() { - if string(registry.Type) == t { - permitted = true - break - } - } - if !permitted { - p.SendBadRequestError(fmt.Errorf("unsupported registry type %s", string(registry.Type))) - return - } - } - - var hardLimits types.ResourceList - if config.QuotaPerProjectEnable() { - setting, err := config.QuotaSetting() - if err != nil { - log.Errorf("failed to get quota setting: %v", err) - p.SendInternalServerError(fmt.Errorf("failed to get quota setting: %v", err)) - return - } - - if !p.SecurityCtx.IsSysAdmin() { - pro.StorageLimit = &setting.StoragePerProject - } - - hardLimits, err = projectQuotaHardLimits(p.Ctx.Request.Context(), pro, setting) - if err != nil { - log.Errorf("Invalid project request, error: %v", err) - p.SendBadRequestError(fmt.Errorf("invalid request: %v", err)) - return - } - } - - exist, err := p.ProjectMgr.Exists(pro.Name) - if err != nil { - p.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s", - pro.Name), err) - return - } - if exist { - p.SendConflictError(errors.New("conflict project")) - return - } - - if pro.Metadata == nil { - pro.Metadata = map[string]string{} - } - // accept the "public" property to make replication work well with old versions(<=1.2.0) - if pro.Public != nil && len(pro.Metadata[models.ProMetaPublic]) == 0 { - pro.Metadata[models.ProMetaPublic] = strconv.FormatBool(*pro.Public == 1) - } - - // populate public metadata as false if it isn't set - if _, ok := pro.Metadata[models.ProMetaPublic]; !ok { - pro.Metadata[models.ProMetaPublic] = strconv.FormatBool(false) - } - // populate - - owner := p.SecurityCtx.GetUsername() - // set the owner as the system admin when the API being called by replication - // it's a solution to workaround the restriction of project creation API: - // only normal users can create projects - if p.SecurityCtx.IsSolutionUser() { - user, err := dao.GetUser(models.User{ - UserID: 1, - }) - if err != nil { - p.SendInternalServerError(fmt.Errorf("failed to get the user 1: %v", err)) - return - } - owner = user.Username - } - projectID, err := p.ProjectMgr.Create(&models.Project{ - Name: pro.Name, - OwnerName: owner, - Metadata: pro.Metadata, - RegistryID: pro.RegistryID, - }) - if err != nil { - if err == errutil.ErrDupProject { - log.Debugf("conflict %s", pro.Name) - p.SendConflictError(fmt.Errorf("conflict %s", pro.Name)) - } else { - p.ParseAndHandleError("failed to add project", err) - } - return - } - - if config.QuotaPerProjectEnable() { - ctx := p.Ctx.Request.Context() - referenceID := quota.ReferenceID(projectID) - if _, err := quota.Ctl.Create(ctx, quota.ProjectReference, referenceID, hardLimits); err != nil { - p.SendInternalServerError(fmt.Errorf("failed to create quota for project: %v", err)) - return - } - } - - // create a default retention policy for proxy project - if pro.RegistryID > 0 { - if err := p.addRetentionPolicyForProxy(projectID); err != nil { - p.SendInternalServerError(fmt.Errorf("failed to add tag retention policy for project: %v", err)) - return - } - } - - // fire event - evt.BuildAndPublish(&metadata.CreateProjectEventMetadata{ - ProjectID: projectID, - Project: pro.Name, - Operator: owner, - }) - - p.Redirect(http.StatusCreated, strconv.FormatInt(projectID, 10)) -} - -func (p *ProjectAPI) addRetentionPolicyForProxy(projID int64) error { - plc := policy.WithNDaysSinceLastPull(projID, defaultDaysToRetention) - retID, err := retentionController.CreateRetention(plc) - if err != nil { - return err - } - if err := p.ProjectMgr.GetMetadataManager().Add(projID, map[string]string{"retention_id": strconv.FormatInt(retID, 10)}); err != nil { - return err - } - return nil -} - -// Head ... -func (p *ProjectAPI) Head() { - - if !p.SecurityCtx.IsAuthenticated() { - p.SendUnAuthorizedError(errors.New("Unauthorized")) - return - } - - name := p.GetString("project_name") - if len(name) == 0 { - p.SendBadRequestError(errors.New("project_name is needed")) - return - } - - project, err := p.ProjectMgr.Get(name) - if err != nil { - p.ParseAndHandleError(fmt.Sprintf("failed to get project %s", name), err) - return - } - - if project == nil { - p.SendNotFoundError(fmt.Errorf("project %s not found", name)) - return - } -} - -// Get ... -func (p *ProjectAPI) Get() { - if !p.requireAccess(rbac.ActionRead) { - return - } - - err := p.populateProperties(p.project) - if err != nil { - log.Errorf("populate project properties failed with : %+v", err) - } - - p.Data["json"] = p.project - p.ServeJSON() -} - -// Delete ... -func (p *ProjectAPI) Delete() { - if !p.requireAccess(rbac.ActionDelete) { - return - } - - result, err := p.deletable(p.project.ProjectID) - if err != nil { - p.SendInternalServerError(fmt.Errorf( - "failed to check the deletable of project %d: %v", p.project.ProjectID, err)) - return - } - if !result.Deletable { - p.SendPreconditionFailedError(errors.New(result.Message)) - return - } - - ctx := p.Ctx.Request.Context() - - if err := project.Ctl.Delete(ctx, p.project.ProjectID); err != nil { - p.ParseAndHandleError(fmt.Sprintf("failed to delete project %d", p.project.ProjectID), err) - return - } - - referenceID := quota.ReferenceID(p.project.ProjectID) - q, err := quota.Ctl.GetByRef(ctx, quota.ProjectReference, referenceID) - if err != nil { - log.Warningf("failed to get quota for project %s, error: %v", p.project.Name, err) - } else { - if err := quota.Ctl.Delete(ctx, q.ID); err != nil { - p.SendInternalServerError(fmt.Errorf("failed to delete quota for project: %v", err)) - return - } - } - - // fire event - evt.BuildAndPublish(&metadata.DeleteProjectEventMetadata{ - ProjectID: p.project.ProjectID, - Project: p.project.Name, - Operator: p.SecurityCtx.GetUsername(), - }) -} - -// Deletable ... -func (p *ProjectAPI) Deletable() { - if !p.requireAccess(rbac.ActionDelete) { - return - } - - result, err := p.deletable(p.project.ProjectID) - if err != nil { - p.SendInternalServerError(fmt.Errorf( - "failed to check the deletable of project %d: %v", p.project.ProjectID, err)) - return - } - - p.Data["json"] = result - p.ServeJSON() -} - -func (p *ProjectAPI) deletable(projectID int64) (*deletableResp, error) { - count, err := dao.GetTotalOfRepositories(&models.RepositoryQuery{ - ProjectIDs: []int64{projectID}, - }) - if err != nil { - return nil, err - } - - if count > 0 { - return &deletableResp{ - Deletable: false, - Message: "the project contains repositories, can not be deleted", - }, nil - } - - // Check helm charts number - if config.WithChartMuseum() { - charts, err := chartController.ListCharts(p.project.Name) - if err != nil { - return nil, err - } - - if len(charts) > 0 { - return &deletableResp{ - Deletable: false, - Message: "the project contains helm charts, can not be deleted", - }, nil - } - } - - return &deletableResp{ - Deletable: true, - }, nil -} - -// List ... -func (p *ProjectAPI) List() { - // query strings - page, size, err := p.GetPaginationParams() - if err != nil { - p.SendBadRequestError(err) - return - } - query := &models.ProjectQueryParam{ - Name: p.GetString("name"), - Owner: p.GetString("owner"), - Pagination: &models.Pagination{ - Page: page, - Size: size, - }, - } - - public := p.GetString("public") - if len(public) > 0 { - pub, err := strconv.ParseBool(public) - if err != nil { - p.SendBadRequestError(fmt.Errorf("invalid public: %s", public)) - return - } - query.Public = &pub - } - - var projects []*models.Project - if !p.SecurityCtx.IsAuthenticated() { - // not login, only get public projects - pros, err := p.ProjectMgr.GetPublic() - if err != nil { - p.SendInternalServerError(fmt.Errorf("failed to get public projects: %v", err)) - return - } - projects = []*models.Project{} - projects = append(projects, pros...) - } else { - if !(p.SecurityCtx.IsSysAdmin() || p.SecurityCtx.IsSolutionUser()) { - projects = []*models.Project{} - // login, but not system admin or solution user, get public projects and - // projects that the user is member of - pros, err := p.ProjectMgr.GetPublic() - if err != nil { - p.SendInternalServerError(fmt.Errorf("failed to get public projects: %v", err)) - return - } - projects = append(projects, pros...) - if sc, ok := p.SecurityCtx.(*local.SecurityContext); ok { - mps, err := p.ProjectMgr.GetAuthorized(sc.User()) - if err != nil { - p.SendInternalServerError(fmt.Errorf("failed to list authorized projects: %v", err)) - return - } - projects = append(projects, mps...) - } - } - } - // Query projects by user group - - if projects != nil { - projectIDs := []int64{} - for _, project := range projects { - projectIDs = append(projectIDs, project.ProjectID) - } - query.ProjectIDs = projectIDs - } - - result, err := p.ProjectMgr.List(query) - if err != nil { - p.ParseAndHandleError("failed to list projects", err) - return - } - - for _, project := range result.Projects { - err = p.populateProperties(project) - if err != nil { - log.Errorf("populate project properties failed %v", err) - } - } - p.SetPaginationHeader(result.Total, page, size) - p.Data["json"] = result.Projects - p.ServeJSON() -} - -func (p *ProjectAPI) populateProperties(project *models.Project) error { - // Transform the severity to severity of CVSS v3.0 Ratings - if severity, ok := project.GetMetadata(models.ProMetaSeverity); ok { - project.SetMetadata(models.ProMetaSeverity, strings.ToLower(vuln.ParseSeverityVersion3(severity).String())) - } - - if sc, ok := p.SecurityCtx.(*local.SecurityContext); ok { - roles, err := pro.ListRoles(sc.User(), project.ProjectID) - if err != nil { - return err - } - project.RoleList = roles - project.Role = highestRole(roles) - } - - total, err := dao.GetTotalOfRepositories(&models.RepositoryQuery{ - ProjectIDs: []int64{project.ProjectID}, - }) - if err != nil { - err = errors.Wrap(err, fmt.Sprintf("get repo count of project %d failed", project.ProjectID)) - return err - } - - project.RepoCount = total - - // Populate chart count property - if config.WithChartMuseum() { - count, err := chartController.GetCountOfCharts([]string{project.Name}) - if err != nil { - err = errors.Wrap(err, fmt.Sprintf("get chart count of project %d failed", project.ProjectID)) - return err - } - - project.ChartCount = count - } - return nil -} - -// Put ... -func (p *ProjectAPI) Put() { - if !p.requireAccess(rbac.ActionUpdate) { - return - } - - var req *models.ProjectRequest - if err := p.DecodeJSONReq(&req); err != nil { - p.SendBadRequestError(err) - return - } - - if err := p.ProjectMgr.Update(p.project.ProjectID, - &models.Project{ - Metadata: req.Metadata, - CVEAllowlist: req.CVEAllowlist, - }); err != nil { - p.ParseAndHandleError(fmt.Sprintf("failed to update project %d", - p.project.ProjectID), err) - return - } -} - -// Summary returns the summary of the project -func (p *ProjectAPI) Summary() { - if !p.requireAccess(rbac.ActionRead) { - return - } - - if err := p.populateProperties(p.project); err != nil { - log.Warningf("populate project properties failed with : %+v", err) - } - - summary := &models.ProjectSummary{ - RepoCount: p.project.RepoCount, - ChartCount: p.project.ChartCount, - } - - var fetchSummaries []func(context.Context, int64, *models.ProjectSummary) - - if hasPerm, _ := p.HasProjectPermission(p.project.ProjectID, rbac.ActionRead, rbac.ResourceQuota); hasPerm { - fetchSummaries = append(fetchSummaries, getProjectQuotaSummary) - } - - if hasPerm, _ := p.HasProjectPermission(p.project.ProjectID, rbac.ActionList, rbac.ResourceMember); hasPerm { - fetchSummaries = append(fetchSummaries, getProjectMemberSummary) - } - - ctx := p.Ctx.Request.Context() - - var wg sync.WaitGroup - for _, fn := range fetchSummaries { - fn := fn - - wg.Add(1) - go func() { - defer wg.Done() - fn(ctx, p.project.ProjectID, summary) - }() - } - wg.Wait() - - if p.project.RegistryID > 0 { - registry, err := replication.RegistryMgr.Get(p.project.RegistryID) - if err != nil { - log.Warningf("failed to get registry %d: %v", p.project.RegistryID, err) - } else { - if registry != nil { - registry.Credential = nil - summary.Registry = registry - } - } - } - - p.Data["json"] = summary - p.ServeJSON() -} - -// TODO move this to pa ckage models -func validateProjectReq(req *models.ProjectRequest) error { - pn := req.Name - if utils.IsIllegalLength(pn, projectNameMinLen, projectNameMaxLen) { - return fmt.Errorf("Project name %s is illegal in length. (greater than %d or less than %d)", pn, projectNameMaxLen, projectNameMinLen) - } - validProjectName := regexp.MustCompile(`^` + restrictedNameChars + `$`) - legal := validProjectName.MatchString(pn) - if !legal { - return fmt.Errorf("project name is not in lower case or contains illegal characters") - } - - metas, err := validateProjectMetadata(req.Metadata) - if err != nil { - return err - } - - req.Metadata = metas - return nil -} - -func projectQuotaHardLimits(ctx context.Context, req *models.ProjectRequest, setting *models.QuotaSetting) (types.ResourceList, error) { - hardLimits := types.ResourceList{} - - if req.StorageLimit != nil { - hardLimits[types.ResourceStorage] = *req.StorageLimit - } else { - hardLimits[types.ResourceStorage] = setting.StoragePerProject - } - - if err := quota.Validate(ctx, quota.ProjectReference, hardLimits); err != nil { - return nil, err - } - - return hardLimits, nil -} - -func getProjectQuotaSummary(ctx context.Context, projectID int64, summary *models.ProjectSummary) { - if !config.QuotaPerProjectEnable() { - log.Debug("Quota per project disabled") - return - } - - q, err := quota.Ctl.GetByRef(ctx, quota.ProjectReference, quota.ReferenceID(projectID)) - if err != nil { - log.Debugf("failed to get quota for project: %d", projectID) - return - } - - summary.Quota = &models.QuotaSummary{} - summary.Quota.Hard, _ = types.NewResourceList(q.Hard) - summary.Quota.Used, _ = types.NewResourceList(q.Used) -} - -func getProjectMemberSummary(ctx context.Context, projectID int64, summary *models.ProjectSummary) { - var wg sync.WaitGroup - - for _, e := range []struct { - role int - count *int64 - }{ - {common.RoleProjectAdmin, &summary.ProjectAdminCount}, - {common.RoleMaintainer, &summary.MaintainerCount}, - {common.RoleDeveloper, &summary.DeveloperCount}, - {common.RoleGuest, &summary.GuestCount}, - {common.RoleLimitedGuest, &summary.LimitedGuestCount}, - } { - wg.Add(1) - go func(role int, count *int64) { - defer wg.Done() - - total, err := pro.GetTotalOfProjectMembers(projectID, role) - if err != nil { - log.Debugf("failed to get total of project members of role %d", role) - return - } - - *count = total - }(e.role, e.count) - } - - wg.Wait() -} - -// Returns the highest role in the role list. -// This func should be removed once we deprecate the "current_user_role_id" in project API -// A user can have multiple roles and they may not have a strict ranking relationship -func highestRole(roles []int) int { - if roles == nil { - return 0 - } - rolePower := map[int]int{ - common.RoleProjectAdmin: 50, - common.RoleMaintainer: 40, - common.RoleDeveloper: 30, - common.RoleGuest: 20, - common.RoleLimitedGuest: 10, - } - var highest, highestPower int - for _, role := range roles { - if p, ok := rolePower[role]; ok && p > highestPower { - highest = role - highestPower = p - } - } - return highest -} diff --git a/src/core/api/project_test.go b/src/core/api/project_test.go deleted file mode 100644 index 17ce38f7a..000000000 --- a/src/core/api/project_test.go +++ /dev/null @@ -1,535 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package api - -import ( - "fmt" - "net/http" - "strconv" - "testing" - - "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/testing/apitests/apilib" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var addProject *apilib.ProjectReq -var addPID int - -func addProjectByName(apiTest *testapi, projectName string) (int32, error) { - req := apilib.ProjectReq{ProjectName: projectName} - code, err := apiTest.ProjectsPost(*admin, req) - if err != nil { - return 0, err - } - if code != http.StatusCreated { - return 0, fmt.Errorf("created failed") - } - - code, projects, err := apiTest.ProjectsGet(&apilib.ProjectQuery{Name: projectName}, *admin) - if err != nil { - return 0, err - } - if code != http.StatusOK { - return 0, fmt.Errorf("get failed") - } - - if len(projects) == 0 { - return 0, fmt.Errorf("oops") - } - - return projects[0].ProjectId, nil -} - -func deleteProjectByIDs(apiTest *testapi, projectIDs ...int32) error { - for _, projectID := range projectIDs { - _, err := apiTest.ProjectsDelete(*admin, fmt.Sprintf("%d", projectID)) - if err != nil { - return err - } - } - - return nil -} - -func InitAddPro() { - addProject = &apilib.ProjectReq{ProjectName: "add_project", Metadata: map[string]string{models.ProMetaPublic: "true"}} -} - -func TestAddProject(t *testing.T) { - - fmt.Println("\nTesting Add Project(ProjectsPost) API") - assert := assert.New(t) - - apiTest := newHarborAPI() - - // prepare for test - InitAddPro() - - // case 1: admin not login, expect project creation fail. - - result, err := apiTest.ProjectsPost(*unknownUsr, *addProject) - if err != nil { - t.Error("Error while creat project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(401), result, "Case 1: Project creation status should be 401") - // t.Log(result) - } - - // case 2: admin successful login, expect project creation success. - fmt.Println("case 2: admin successful login, expect project creation success.") - - result, err = apiTest.ProjectsPost(*admin, *addProject) - if err != nil { - t.Error("Error while creat project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(201), result, "Case 2: Project creation status should be 201") - // t.Log(result) - } - - // case 3: duplicate project name, create project fail - fmt.Println("case 3: duplicate project name, create project fail") - - result, err = apiTest.ProjectsPost(*admin, *addProject) - if err != nil { - t.Error("Error while creat project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(409), result, "Case 3: Project creation status should be 409") - // t.Log(result) - } - - // case 4: response code = 400 : Project name is illegal in length - fmt.Println("case 4 : response code = 400 : Project name is illegal in length ") - - result, err = apiTest.ProjectsPost(*admin, apilib.ProjectReq{ProjectName: "", Metadata: map[string]string{models.ProMetaPublic: "true"}}) - if err != nil { - t.Error("Error while creat project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(400), result, "case 4 : response code = 400 : Project name is illegal in length ") - } - - // case 5: response code = 201 : expect project creation with quota success. - fmt.Println("case 5 : response code = 201 : expect project creation with quota success ") - - var countLimit, storageLimit int64 - countLimit, storageLimit = 100, 10 - result, err = apiTest.ProjectsPost(*admin, apilib.ProjectReq{ProjectName: "with_quota", CountLimit: &countLimit, StorageLimit: &storageLimit}) - if err != nil { - t.Error("Error while creat project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(201), result, "case 5 : response code = 201 : expect project creation with quota success ") - } - - // case 6: response code = 400 : bad quota value, create project fail - fmt.Println("case 6: response code = 400 : bad quota value, create project fail") - - countLimit, storageLimit = 100, -2 - result, err = apiTest.ProjectsPost(*admin, apilib.ProjectReq{ProjectName: "with_quota", CountLimit: &countLimit, StorageLimit: &storageLimit}) - if err != nil { - t.Error("Error while creat project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(400), result, "case 6: response code = 400 : bad quota value, create project fail") - } - - fmt.Printf("\n") - -} - -func TestListProjects(t *testing.T) { - fmt.Println("\nTest for Project GET API by project name") - assert := assert.New(t) - - apiTest := newHarborAPI() - var result []apilib.Project - - cMockServer, oldCtrl, err := mockChartController() - if err != nil { - t.Fatal(err) - } - defer func() { - cMockServer.Close() - chartController = oldCtrl - }() - - // ----------------------------case 1 : Response Code=200----------------------------// - fmt.Println("case 1: response code:200") - httpStatusCode, result, err := apiTest.ProjectsGet( - &apilib.ProjectQuery{ - Name: addProject.ProjectName, - Owner: admin.Name, - Public: true, - }) - assert.Nil(err) - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - assert.Equal(addProject.ProjectName, result[0].ProjectName, "Project name is wrong") - assert.Equal("true", result[0].Metadata[models.ProMetaPublic], "Public is wrong") - - // find add projectID - addPID = int(result[0].ProjectId) - - // -------------------case 3 : check admin project role------------------------// - httpStatusCode, result, err = apiTest.ProjectsGet( - &apilib.ProjectQuery{ - Name: addProject.ProjectName, - Owner: admin.Name, - Public: true, - }, *admin) - if err != nil { - t.Error("Error while search project by proName and isPublic", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - assert.Equal(addProject.ProjectName, result[0].ProjectName, "Project name is wrong") - assert.Equal("true", result[0].Metadata[models.ProMetaPublic], "Public is wrong") - assert.Equal(int32(1), result[0].CurrentUserRoleId, "User project role is wrong") - } - - // -------------------case 4 : add project member and check his role ------------------------// - CommonAddUser() - member := &models.MemberReq{ - Role: 2, - MemberUser: models.User{ - Username: TestUserName, - }, - } - projectID := strconv.Itoa(addPID) - var memberID int - httpStatusCode, memberID, err = apiTest.AddProjectMember(*admin, projectID, member) - if err != nil { - t.Error("Error whihle add project role member", err.Error()) - t.Log(err) - } else { - assert.Equal(int(201), httpStatusCode, "httpStatusCode should be 201") - } - httpStatusCode, result, err = apiTest.ProjectsGet( - &apilib.ProjectQuery{ - Name: addProject.ProjectName, - }, *testUser) - if err != nil { - t.Error("Error while search project by proName and isPublic", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - assert.Equal(addProject.ProjectName, result[0].ProjectName, "Project name is wrong") - assert.Equal("true", result[0].Metadata[models.ProMetaPublic], "Public is wrong") - assert.Equal(int32(2), result[0].CurrentUserRoleId, "User project role is wrong") - } - httpStatusCode, err = apiTest.DeleteProjectMember(*admin, projectID, strconv.Itoa(memberID)) - if err != nil { - t.Error("Error whihle add project role member", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - } - CommonDelUser() -} - -// Get project by proID -func TestProGetByID(t *testing.T) { - fmt.Println("\nTest for Project GET API by project id") - assert := assert.New(t) - - apiTest := newHarborAPI() - var result apilib.Project - projectID := strconv.Itoa(addPID) - - cMockServer, oldCtrl, err := mockChartController() - if err != nil { - t.Fatal(err) - } - defer func() { - cMockServer.Close() - chartController = oldCtrl - }() - - // ----------------------------case 1 : Response Code=200----------------------------// - fmt.Println("case 1: response code:200") - httpStatusCode, result, err := apiTest.ProjectsGetByPID(projectID) - if err != nil { - t.Error("Error while search project by proID", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - assert.Equal(addProject.ProjectName, result.ProjectName, "ProjectName is wrong") - assert.Equal("true", result.Metadata[models.ProMetaPublic], "Public is wrong") - } - fmt.Printf("\n") -} -func TestDeleteProject(t *testing.T) { - - fmt.Println("\nTesting Delete Project(ProjectsPost) API") - assert := assert.New(t) - - apiTest := newHarborAPI() - - projectID := strconv.Itoa(addPID) - - // --------------------------case 1: Response Code=401,User need to log in first.-----------------------// - fmt.Println("case 1: Response Code=401,User need to log in first.") - httpStatusCode, err := apiTest.ProjectsDelete(*unknownUsr, projectID) - if err != nil { - t.Error("Error while delete project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(401), httpStatusCode, "Case 1: Project deletion status should be 401") - } - - // --------------------------case 2: Response Code=200---------------------------------// - fmt.Println("case2: response code:200") - httpStatusCode, err = apiTest.ProjectsDelete(*admin, projectID) - if err != nil { - t.Error("Error while delete project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "Case 2: Project deletion status should be 200") - } - - // --------------------------case 3: Response Code=404,Project does not exist---------------------------------// - fmt.Println("case 3: Response Code=404,Project does not exist") - projectID = "11" - httpStatusCode, err = apiTest.ProjectsDelete(*admin, projectID) - if err != nil { - t.Error("Error while delete project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(404), httpStatusCode, "Case 3: Project deletion status should be 404") - } - - // --------------------------case 4: Response Code=400,Invalid project id.---------------------------------// - fmt.Println("case 4: Response Code=400,Invalid project id.") - projectID = "cc" - httpStatusCode, err = apiTest.ProjectsDelete(*admin, projectID) - if err != nil { - t.Error("Error while delete project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(400), httpStatusCode, "Case 4: Project deletion status should be 400") - } - fmt.Printf("\n") - -} -func TestProHead(t *testing.T) { - t.Log("\nTest for Project HEAD API") - assert := assert.New(t) - - apiTest := newHarborAPI() - - // ----------------------------case 1 : Response Code=200----------------------------// - t.Log("case 1: response code:200") - httpStatusCode, err := apiTest.ProjectsHead(*admin, "library") - if err != nil { - t.Error("Error while search project by proName", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - } - - // ----------------------------case 2 : Response Code=404:Project name does not exist.----------------------------// - t.Log("case 2: response code:404,Project name does not exist.") - httpStatusCode, err = apiTest.ProjectsHead(*admin, "libra") - if err != nil { - t.Error("Error while search project by proName", err.Error()) - t.Log(err) - } else { - assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404") - } - - t.Log("case 3: response code:401. Project exist with unauthenticated user") - httpStatusCode, err = apiTest.ProjectsHead(*unknownUsr, "library") - if err != nil { - t.Error("Error while search project by proName", err.Error()) - t.Log(err) - } else { - assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 404") - } - - t.Log("case 4: response code:401. Project name does not exist with unauthenticated user") - httpStatusCode, err = apiTest.ProjectsHead(*unknownUsr, "libra") - if err != nil { - t.Error("Error while search project by proName", err.Error()) - t.Log(err) - } else { - assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 404") - } - - fmt.Printf("\n") -} - -func TestPut(t *testing.T) { - fmt.Println("\nTest for Project PUT API: Update properties for a selected project") - assert := assert.New(t) - - apiTest := newHarborAPI() - - project := &models.Project{ - Metadata: map[string]string{ - models.ProMetaPublic: "true", - }, - } - - fmt.Println("case 1: response code:200") - code, err := apiTest.ProjectsPut(*admin, "1", project) - require.Nil(t, err) - assert.Equal(int(200), code) - - fmt.Println("case 2: response code:401, User need to log in first.") - code, err = apiTest.ProjectsPut(*unknownUsr, "1", project) - require.Nil(t, err) - assert.Equal(int(401), code) - - fmt.Println("case 3: response code:400, Invalid project id") - code, err = apiTest.ProjectsPut(*admin, "cc", project) - require.Nil(t, err) - assert.Equal(int(400), code) - - fmt.Println("case 4: response code:404, Not found the project") - code, err = apiTest.ProjectsPut(*admin, "1234", project) - require.Nil(t, err) - assert.Equal(int(404), code) - - fmt.Printf("\n") -} - -func TestDeletable(t *testing.T) { - apiTest := newHarborAPI() - chServer, oldController, err := mockChartController() - require.Nil(t, err) - require.NotNil(t, chServer) - defer chServer.Close() - defer func() { - chartController = oldController - }() - - project := models.Project{ - Name: "project_for_test_deletable", - OwnerID: 1, - } - id, err := dao.AddProject(project) - require.Nil(t, err) - - // non-exist project - code, del, err := apiTest.ProjectDeletable(*admin, 1000) - assert.Nil(t, err) - assert.Equal(t, http.StatusNotFound, code) - - // unauthorized - code, del, err = apiTest.ProjectDeletable(*unknownUsr, id) - assert.Nil(t, err) - assert.Equal(t, http.StatusUnauthorized, code) - - // can be deleted - code, del, err = apiTest.ProjectDeletable(*admin, id) - assert.Nil(t, err) - assert.Equal(t, http.StatusOK, code) - assert.True(t, del) - - err = dao.AddRepository(models.RepoRecord{ - Name: project.Name + "/golang", - ProjectID: id, - }) - require.Nil(t, err) - - // can not be deleted as contains repository - code, del, err = apiTest.ProjectDeletable(*admin, id) - assert.Nil(t, err) - assert.Equal(t, http.StatusOK, code) - assert.False(t, del) -} - -func TestProjectSummary(t *testing.T) { - fmt.Println("\nTest for Project Summary API") - assert := assert.New(t) - - apiTest := newHarborAPI() - - projectID, err := addProjectByName(apiTest, "project-summary") - assert.Nil(err) - defer func() { - deleteProjectByIDs(apiTest, projectID) - }() - - // ----------------------------case 1 : Response Code=200----------------------------// - fmt.Println("case 1: response code:200") - httpStatusCode, summary, err := apiTest.ProjectSummary(*admin, fmt.Sprintf("%d", projectID)) - if err != nil { - t.Error("Error while search project by proName", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - assert.Equal(int64(1), summary.ProjectAdminCount) - assert.Equal(map[string]int64{"storage": -1}, summary.Quota.Hard) - } - - fmt.Printf("\n") -} - -func TestHighestRole(t *testing.T) { - cases := []struct { - input []int - expect int - }{ - { - []int{}, - 0, - }, - { - []int{ - common.RoleDeveloper, - common.RoleMaintainer, - common.RoleLimitedGuest, - }, - common.RoleMaintainer, - }, - { - []int{ - common.RoleProjectAdmin, - common.RoleMaintainer, - common.RoleMaintainer, - }, - common.RoleProjectAdmin, - }, - { - []int{ - 99, - 33, - common.RoleLimitedGuest, - }, - common.RoleLimitedGuest, - }, - { - []int{ - 99, - 99, - 99, - }, - 0, - }, - { - nil, - 0, - }, - } - for _, c := range cases { - assert.Equal(t, c.expect, highestRole(c.input)) - } -} diff --git a/src/core/api/search.go b/src/core/api/search.go index 9b8ee03f4..88190ae98 100644 --- a/src/core/api/search.go +++ b/src/core/api/search.go @@ -20,6 +20,7 @@ import ( "helm.sh/helm/v3/cmd/helm/search" + "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" pro "github.com/goharbor/harbor/src/common/dao/project" "github.com/goharbor/harbor/src/common/models" @@ -205,3 +206,27 @@ func filterRepositories(projects []*models.Project, keyword string) ( } return result, nil } + +// Returns the highest role in the role list. +// This func should be removed once we deprecate the "current_user_role_id" in project API +// A user can have multiple roles and they may not have a strict ranking relationship +func highestRole(roles []int) int { + if roles == nil { + return 0 + } + rolePower := map[int]int{ + common.RoleProjectAdmin: 50, + common.RoleMaintainer: 40, + common.RoleDeveloper: 30, + common.RoleGuest: 20, + common.RoleLimitedGuest: 10, + } + var highest, highestPower int + for _, role := range roles { + if p, ok := rolePower[role]; ok && p > highestPower { + highest = role + highestPower = p + } + } + return highest +} diff --git a/src/jobservice/job/impl/gc/garbage_collection.go b/src/jobservice/job/impl/gc/garbage_collection.go index e1f26675e..cba55179d 100644 --- a/src/jobservice/job/impl/gc/garbage_collection.go +++ b/src/jobservice/job/impl/gc/garbage_collection.go @@ -18,20 +18,18 @@ import ( "os" "time" - "github.com/goharbor/harbor/src/lib/errors" - redislib "github.com/goharbor/harbor/src/lib/redis" - "github.com/goharbor/harbor/src/pkg/artifactrash/model" - blob_models "github.com/goharbor/harbor/src/pkg/blob/models" - - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/registryctl" "github.com/goharbor/harbor/src/controller/artifact" "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/q" + redislib "github.com/goharbor/harbor/src/lib/redis" "github.com/goharbor/harbor/src/pkg/artifactrash" + "github.com/goharbor/harbor/src/pkg/artifactrash/model" "github.com/goharbor/harbor/src/pkg/blob" + blob_models "github.com/goharbor/harbor/src/pkg/blob/models" "github.com/goharbor/harbor/src/registryctl/client" ) @@ -52,7 +50,6 @@ type GarbageCollector struct { artCtl artifact.Controller artrashMgr artifactrash.Manager blobMgr blob.Manager - projectCtl project.Controller registryCtlClient client.Client logger logger.Interface redisURL string @@ -103,7 +100,6 @@ func (gc *GarbageCollector) init(ctx job.Context, params job.Parameters) error { gc.artCtl = artifact.Ctl gc.artrashMgr = artifactrash.NewManager() gc.blobMgr = blob.NewManager() - gc.projectCtl = project.Ctl } if err := gc.registryCtlClient.Health(); err != nil { gc.logger.Errorf("failed to start gc as registry controller is unreachable: %v", err) @@ -416,50 +412,21 @@ func (gc *GarbageCollector) deletedArt(ctx job.Context) (map[string][]model.Arti // clean the untagged blobs in each project, these blobs are not referenced by any manifest and will be cleaned by GC func (gc *GarbageCollector) removeUntaggedBlobs(ctx job.Context) { - // get all projects - projects := func(chunkSize int) <-chan *models.Project { - ch := make(chan *models.Project, chunkSize) - - go func() { - defer close(ch) - - params := &models.ProjectQueryParam{ - Pagination: &models.Pagination{Page: 1, Size: int64(chunkSize)}, - } - - for { - results, err := gc.projectCtl.List(ctx.SystemContext(), params, project.Metadata(false)) - if err != nil { - gc.logger.Errorf("list projects failed, error: %v", err) - return - } - - for _, p := range results { - ch <- p - } - - if len(results) < chunkSize { - break - } - - params.Pagination.Page++ - } - - }() - - return ch - }(50) - - for project := range projects { + for result := range project.ListAll(ctx.SystemContext(), 50, nil, project.Metadata(false)) { + if result.Error != nil { + gc.logger.Errorf("remove untagged blobs for all projects got error: %v", result.Error) + continue + } + p := result.Data all, err := gc.blobMgr.List(ctx.SystemContext(), blob.ListParams{ - ProjectID: project.ProjectID, + ProjectID: p.ProjectID, UpdateTime: time.Now().Add(-time.Duration(gc.timeWindowHours) * time.Hour), }) if err != nil { gc.logger.Errorf("failed to get blobs of project, %v", err) continue } - if err := gc.blobMgr.CleanupAssociationsForProject(ctx.SystemContext(), project.ProjectID, all); err != nil { + if err := gc.blobMgr.CleanupAssociationsForProject(ctx.SystemContext(), p.ProjectID, all); err != nil { gc.logger.Errorf("failed to clean untagged blobs of project, %v", err) continue } diff --git a/src/jobservice/job/impl/gc/garbage_collection_test.go b/src/jobservice/job/impl/gc/garbage_collection_test.go index 95a39d224..fd80272b9 100644 --- a/src/jobservice/job/impl/gc/garbage_collection_test.go +++ b/src/jobservice/job/impl/gc/garbage_collection_test.go @@ -1,9 +1,27 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package gc import ( + "os" + "testing" + "github.com/docker/distribution/manifest/schema2" "github.com/goharbor/harbor/src/common/models" commom_regctl "github.com/goharbor/harbor/src/common/registryctl" + "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/artifact" "github.com/goharbor/harbor/src/pkg/artifactrash/model" @@ -17,8 +35,6 @@ import ( "github.com/goharbor/harbor/src/testing/pkg/blob" "github.com/goharbor/harbor/src/testing/registryctl" "github.com/stretchr/testify/suite" - "os" - "testing" ) type gcTestSuite struct { @@ -29,6 +45,8 @@ type gcTestSuite struct { projectCtl *projecttesting.Controller blobMgr *blob.Manager + originalProjectCtl project.Controller + regCtlInit func() } @@ -39,9 +57,16 @@ func (suite *gcTestSuite) SetupTest() { suite.blobMgr = &blob.Manager{} suite.projectCtl = &projecttesting.Controller{} + suite.originalProjectCtl = project.Ctl + project.Ctl = suite.projectCtl + regCtlInit = func() { commom_regctl.RegistryCtlClient = suite.registryCtlClient } } +func (suite *gcTestSuite) TearDownTest() { + project.Ctl = suite.originalProjectCtl +} + func (suite *gcTestSuite) TestMaxFails() { gc := &GarbageCollector{} suite.Equal(uint(1), gc.MaxFails()) @@ -110,8 +135,7 @@ func (suite *gcTestSuite) TestRemoveUntaggedBlobs() { mock.OnAnything(suite.blobMgr, "CleanupAssociationsForProject").Return(nil) gc := &GarbageCollector{ - projectCtl: suite.projectCtl, - blobMgr: suite.blobMgr, + blobMgr: suite.blobMgr, } suite.NotPanics(func() { @@ -244,7 +268,6 @@ func (suite *gcTestSuite) TestRun() { gc := &GarbageCollector{ artCtl: suite.artifactCtl, artrashMgr: suite.artrashMgr, - projectCtl: suite.projectCtl, blobMgr: suite.blobMgr, registryCtlClient: suite.registryCtlClient, } @@ -318,7 +341,6 @@ func (suite *gcTestSuite) TestMark() { gc := &GarbageCollector{ artCtl: suite.artifactCtl, artrashMgr: suite.artrashMgr, - projectCtl: suite.projectCtl, blobMgr: suite.blobMgr, } @@ -336,7 +358,6 @@ func (suite *gcTestSuite) TestSweep() { gc := &GarbageCollector{ artCtl: suite.artifactCtl, artrashMgr: suite.artrashMgr, - projectCtl: suite.projectCtl, blobMgr: suite.blobMgr, registryCtlClient: suite.registryCtlClient, deleteSet: []*pkg_blob.Blob{ diff --git a/src/jobservice/job/impl/gcreadonly/garbage_collection.go b/src/jobservice/job/impl/gcreadonly/garbage_collection.go index 61a750400..def10758d 100644 --- a/src/jobservice/job/impl/gcreadonly/garbage_collection.go +++ b/src/jobservice/job/impl/gcreadonly/garbage_collection.go @@ -16,22 +16,20 @@ package gcreadonly import ( "fmt" - redislib "github.com/goharbor/harbor/src/lib/redis" "os" "time" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/controller/artifact" - "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/lib/q" - "github.com/goharbor/harbor/src/pkg/artifactrash" - "github.com/goharbor/harbor/src/pkg/blob" - "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/config" "github.com/goharbor/harbor/src/common/registryctl" + "github.com/goharbor/harbor/src/controller/artifact" + "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/lib/q" + redislib "github.com/goharbor/harbor/src/lib/redis" + "github.com/goharbor/harbor/src/pkg/artifactrash" + "github.com/goharbor/harbor/src/pkg/blob" "github.com/goharbor/harbor/src/registryctl/client" ) @@ -160,7 +158,6 @@ func (gc *GarbageCollector) init(ctx job.Context, params job.Parameters) error { gc.artCtl = artifact.Ctl gc.artrashMgr = artifactrash.NewManager() gc.blobMgr = blob.NewManager() - gc.projectCtl = project.Ctl } if err := gc.registryCtlClient.Health(); err != nil { gc.logger.Errorf("failed to start gc as registry controller is unreachable: %v", err) @@ -285,49 +282,20 @@ func (gc *GarbageCollector) deleteCandidates(ctx job.Context) error { // clean the untagged blobs in each project, these blobs are not referenced by any manifest and will be cleaned by GC func (gc *GarbageCollector) removeUntaggedBlobs(ctx job.Context) { - // get all projects - projects := func(chunkSize int) <-chan *models.Project { - ch := make(chan *models.Project, chunkSize) - - go func() { - defer close(ch) - - params := &models.ProjectQueryParam{ - Pagination: &models.Pagination{Page: 1, Size: int64(chunkSize)}, - } - - for { - results, err := gc.projectCtl.List(ctx.SystemContext(), params, project.Metadata(false)) - if err != nil { - gc.logger.Errorf("list projects failed, error: %v", err) - return - } - - for _, p := range results { - ch <- p - } - - if len(results) < chunkSize { - break - } - - params.Pagination.Page++ - } - - }() - - return ch - }(50) - - for project := range projects { + for result := range project.ListAll(ctx.SystemContext(), 50, nil, project.Metadata(false)) { + if result.Error != nil { + gc.logger.Errorf("remove untagged blobs for all projects got error: %v", result.Error) + continue + } + p := result.Data all, err := gc.blobMgr.List(ctx.SystemContext(), blob.ListParams{ - ProjectID: project.ProjectID, + ProjectID: p.ProjectID, }) if err != nil { gc.logger.Errorf("failed to get blobs of project, %v", err) continue } - if err := gc.blobMgr.CleanupAssociationsForProject(ctx.SystemContext(), project.ProjectID, all); err != nil { + if err := gc.blobMgr.CleanupAssociationsForProject(ctx.SystemContext(), p.ProjectID, all); err != nil { gc.logger.Errorf("failed to clean untagged blobs of project, %v", err) continue } diff --git a/src/jobservice/job/impl/gcreadonly/garbage_collection_test.go b/src/jobservice/job/impl/gcreadonly/garbage_collection_test.go index 378be51be..f04ecb582 100644 --- a/src/jobservice/job/impl/gcreadonly/garbage_collection_test.go +++ b/src/jobservice/job/impl/gcreadonly/garbage_collection_test.go @@ -1,9 +1,27 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package gcreadonly import ( + "os" + "testing" + "github.com/goharbor/harbor/src/common/config" "github.com/goharbor/harbor/src/common/models" commom_regctl "github.com/goharbor/harbor/src/common/registryctl" + "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/artifact" "github.com/goharbor/harbor/src/pkg/artifactrash/model" @@ -16,8 +34,6 @@ import ( "github.com/goharbor/harbor/src/testing/pkg/blob" "github.com/goharbor/harbor/src/testing/registryctl" "github.com/stretchr/testify/suite" - "os" - "testing" ) type gcTestSuite struct { @@ -28,6 +44,8 @@ type gcTestSuite struct { projectCtl *projecttesting.Controller blobMgr *blob.Manager + originalProjectCtl project.Controller + regCtlInit func() setReadOnly func(cfgMgr *config.CfgManager, switcher bool) error getReadOnly func(cfgMgr *config.CfgManager) (bool, error) @@ -40,11 +58,18 @@ func (suite *gcTestSuite) SetupTest() { suite.blobMgr = &blob.Manager{} suite.projectCtl = &projecttesting.Controller{} + suite.originalProjectCtl = project.Ctl + project.Ctl = suite.projectCtl + regCtlInit = func() { commom_regctl.RegistryCtlClient = suite.registryCtlClient } setReadOnly = func(cfgMgr *config.CfgManager, switcher bool) error { return nil } getReadOnly = func(cfgMgr *config.CfgManager) (bool, error) { return true, nil } } +func (suite *gcTestSuite) TearDownTest() { + project.Ctl = suite.originalProjectCtl +} + func (suite *gcTestSuite) TestMaxFails() { gc := &GarbageCollector{} suite.Equal(uint(1), gc.MaxFails()) @@ -105,8 +130,7 @@ func (suite *gcTestSuite) TestRemoveUntaggedBlobs() { mock.OnAnything(suite.blobMgr, "CleanupAssociationsForProject").Return(nil) gc := &GarbageCollector{ - projectCtl: suite.projectCtl, - blobMgr: suite.blobMgr, + blobMgr: suite.blobMgr, } suite.NotPanics(func() { @@ -218,7 +242,6 @@ func (suite *gcTestSuite) TestRun() { artCtl: suite.artifactCtl, artrashMgr: suite.artrashMgr, cfgMgr: config.NewInMemoryManager(), - projectCtl: suite.projectCtl, blobMgr: suite.blobMgr, registryCtlClient: suite.registryCtlClient, } diff --git a/src/lib/convert_types.go b/src/lib/convert_types.go new file mode 100644 index 000000000..994a998b0 --- /dev/null +++ b/src/lib/convert_types.go @@ -0,0 +1,62 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lib + +import ( + "strconv" +) + +// BoolValue returns the value of the bool pointer or false if the pointer is nil +func BoolValue(v *bool) bool { + if v != nil { + return *v + } + return false +} + +// Int64Value returns the value of the int64 pointer or 0 if the pointer is nil +func Int64Value(v *int64) int64 { + if v != nil { + return *v + } + return 0 +} + +// StringValue returns the value of the string pointer or "" if the pointer is nil +func StringValue(v *string) string { + if v != nil { + return *v + } + return "" +} + +// ToBool convert interface to bool +func ToBool(v interface{}) bool { + switch b := v.(type) { + case bool: + return b + case nil: + return false + case int: + return v.(int) != 0 + case int64: + return v.(int64) != 0 + case string: + r, _ := strconv.ParseBool(v.(string)) + return r + default: + return false + } +} diff --git a/src/lib/convert_types_test.go b/src/lib/convert_types_test.go new file mode 100644 index 000000000..08a9e9f2d --- /dev/null +++ b/src/lib/convert_types_test.go @@ -0,0 +1,51 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lib + +import ( + "testing" +) + +func TestToBool(t *testing.T) { + type args struct { + v interface{} + } + tests := []struct { + name string + args args + want bool + }{ + {"nil", args{nil}, false}, + {"bool true", args{true}, true}, + {"bool false", args{false}, false}, + {"string true", args{"true"}, true}, + {"string True", args{"True"}, true}, + {"string 1", args{"1"}, true}, + {"string false", args{"false"}, false}, + {"string False", args{"False"}, false}, + {"string 0", args{"0"}, false}, + {"int 1", args{1}, true}, + {"int 0", args{0}, false}, + {"int64 1", args{int64(1)}, true}, + {"int64 0", args{int64(0)}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToBool(tt.args.v); got != tt.want { + t.Errorf("ToBool() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/lib/json_copy.go b/src/lib/json_copy.go new file mode 100644 index 000000000..b7cf4a57f --- /dev/null +++ b/src/lib/json_copy.go @@ -0,0 +1,31 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lib + +import ( + "encoding/json" +) + +// JSONCopy copy from src to dst with json marshal and unmarshal. +// NOTE: copy from one struct to another struct may miss some values depend on +// the json tag in the fields, you should know what will happened when call this function. +func JSONCopy(dst, src interface{}) error { + data, err := json.Marshal(src) + if err != nil { + return err + } + + return json.Unmarshal(data, dst) +} diff --git a/src/lib/json_copy_test.go b/src/lib/json_copy_test.go new file mode 100644 index 000000000..4ada47dfe --- /dev/null +++ b/src/lib/json_copy_test.go @@ -0,0 +1,68 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lib + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +type jsonCopyFoo struct { + Name string `json:"name"` + Age int `json:"age"` +} + +func TestJSONCopy(t *testing.T) { + assert := assert.New(t) + + { + var m map[string]interface{} + foo := &jsonCopyFoo{ + Name: "foo", + Age: 1, + } + + assert.Nil(m) + assert.Nil(JSONCopy(&m, foo)) + assert.NotNil(m) + assert.Len(m, 2) + } + + { + var m map[string]interface{} + var foo *jsonCopyFoo + + assert.Nil(m) + assert.Nil(JSONCopy(&m, foo)) + assert.Nil(m) + } + + { + m := map[string]interface{}{ + "name": "foo", + "age": 1, + } + var foo *jsonCopyFoo + assert.Nil(JSONCopy(&foo, &m)) + assert.NotNil(foo) + assert.Equal("foo", foo.Name) + assert.Equal(1, foo.Age) + } + + { + assert.Error(JSONCopy(nil, JSONCopy)) + } +} diff --git a/src/lib/orm/orm.go b/src/lib/orm/orm.go index 351c77e7a..123f5da42 100644 --- a/src/lib/orm/orm.go +++ b/src/lib/orm/orm.go @@ -51,6 +51,11 @@ func Context() context.Context { return NewContext(context.Background(), orm.NewOrm()) } +// Clone returns new context with orm for ctx +func Clone(ctx context.Context) context.Context { + return NewContext(ctx, orm.NewOrm()) +} + // WithTransaction a decorator which make f run in transaction func WithTransaction(f func(ctx context.Context) error) func(ctx context.Context) error { return func(ctx context.Context) error { diff --git a/src/migration/artifact.go b/src/migration/artifact.go index 2d29b6fcd..c3434554c 100644 --- a/src/migration/artifact.go +++ b/src/migration/artifact.go @@ -27,7 +27,7 @@ import ( func abstractArtData(ctx context.Context) error { abstractor := art.NewAbstractor() - pros, err := project.Mgr.List(ctx) + pros, err := project.Mgr.List(ctx, nil) if err != nil { return err } diff --git a/src/pkg/project/dao/dao.go b/src/pkg/project/dao/dao.go index 8be4555b8..4a6533ee7 100644 --- a/src/pkg/project/dao/dao.go +++ b/src/pkg/project/dao/dao.go @@ -94,6 +94,7 @@ func (d *dao) Create(ctx context.Context, project *models.Project) (int64, error func (d *dao) Count(ctx context.Context, query *q.Query) (total int64, err error) { query = q.MustClone(query) query.Keywords["deleted"] = false + query.Sorting = "" query.PageNumber = 0 query.PageSize = 0 @@ -101,6 +102,7 @@ func (d *dao) Count(ctx context.Context, query *q.Query) (total int64, err error if err != nil { return 0, err } + return qs.Count() } @@ -160,6 +162,10 @@ func (d *dao) List(ctx context.Context, query *q.Query) ([]*models.Project, erro return nil, err } + if query.Sorting != "" { + qs = qs.OrderBy(query.Sorting) + } + projects := []*models.Project{} if _, err := qs.All(&projects); err != nil { return nil, err diff --git a/src/pkg/project/dao/dao_test.go b/src/pkg/project/dao/dao_test.go index 03832c671..50f29fa70 100644 --- a/src/pkg/project/dao/dao_test.go +++ b/src/pkg/project/dao/dao_test.go @@ -196,9 +196,11 @@ func (suite *DaoTestSuite) TestList() { } }() - projects, err := suite.dao.List(orm.Context(), q.New(q.KeyWords{"project_id__in": projectIDs})) - suite.Nil(err) - suite.Len(projects, len(projectNames)) + { + projects, err := suite.dao.List(orm.Context(), q.New(q.KeyWords{"project_id__in": projectIDs})) + suite.Nil(err) + suite.Len(projects, len(projectNames)) + } } func (suite *DaoTestSuite) TestListByPublic() { @@ -259,6 +261,13 @@ func (suite *DaoTestSuite) TestListByMember() { suite.Len(projects, 0) } + { + // guest with public projects + projects, err := suite.dao.List(orm.Context(), q.New(q.KeyWords{"member": &models.MemberQuery{Name: "admin", Role: common.RoleGuest, WithPublic: true}})) + suite.Nil(err) + suite.Len(projects, 1) + } + { suite.WithUser(func(userID int64, username string) { project := &models.Project{ diff --git a/src/pkg/project/manager.go b/src/pkg/project/manager.go index fddda6d1d..353defeb8 100644 --- a/src/pkg/project/manager.go +++ b/src/pkg/project/manager.go @@ -16,7 +16,9 @@ package project import ( "context" + "regexp" + "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/project/dao" @@ -43,7 +45,7 @@ type Manager interface { Get(ctx context.Context, idOrName interface{}) (*models.Project, error) // List projects according to the query - List(ctx context.Context, query ...*models.ProjectQueryParam) ([]*models.Project, error) + List(ctx context.Context, query *q.Query) ([]*models.Project, error) } // New returns a default implementation of Manager @@ -51,6 +53,14 @@ func New() Manager { return &manager{dao: dao.New()} } +const projectNameMaxLen int = 255 +const projectNameMinLen int = 1 +const restrictedNameChars = `[a-z0-9]+(?:[._-][a-z0-9]+)*` + +var ( + validProjectName = regexp.MustCompile(`^` + restrictedNameChars + `$`) +) + type manager struct { dao dao.DAO } @@ -60,6 +70,17 @@ func (m *manager) Create(ctx context.Context, project *models.Project) (int64, e if project.OwnerID <= 0 { return 0, errors.BadRequestError(nil).WithMessage("Owner is missing when creating project %s", project.Name) } + + if utils.IsIllegalLength(project.Name, projectNameMinLen, projectNameMaxLen) { + format := "Project name %s is illegal in length. (greater than %d or less than %d)" + return 0, errors.BadRequestError(nil).WithMessage(format, project.Name, projectNameMaxLen, projectNameMinLen) + } + + legal := validProjectName.MatchString(project.Name) + if !legal { + return 0, errors.BadRequestError(nil).WithMessage("project name is not in lower case or contains illegal characters") + } + return m.dao.Create(ctx, project) } @@ -87,14 +108,6 @@ func (m *manager) Get(ctx context.Context, idOrName interface{}) (*models.Projec } // List projects according to the query -func (m *manager) List(ctx context.Context, query ...*models.ProjectQueryParam) ([]*models.Project, error) { - var param *models.ProjectQueryParam - if len(query) > 0 { - param = query[0] - } - if param == nil { - return m.dao.List(ctx, nil) - } - - return m.dao.List(ctx, param.ToQuery()) +func (m *manager) List(ctx context.Context, query *q.Query) ([]*models.Project, error) { + return m.dao.List(ctx, query) } diff --git a/src/pkg/project/models/project.go b/src/pkg/project/models/project.go index 209028d09..d3a9146d4 100644 --- a/src/pkg/project/models/project.go +++ b/src/pkg/project/models/project.go @@ -36,8 +36,5 @@ func (projects Projects) OwnerIDs() []int { // Member ... type Member = models.Member -// ProjectQueryParam ... -type ProjectQueryParam = models.ProjectQueryParam - // MemberQuery ... type MemberQuery = models.MemberQuery diff --git a/src/pkg/retention/launcher.go b/src/pkg/retention/launcher.go index 95164bd66..7b81d2fae 100644 --- a/src/pkg/retention/launcher.go +++ b/src/pkg/retention/launcher.go @@ -326,7 +326,7 @@ func launcherError(err error) error { } func getProjects(projectMgr project.Manager) ([]*selector.Candidate, error) { - projects, err := projectMgr.List(orm.Context()) + projects, err := projectMgr.List(orm.Context(), nil) if err != nil { return nil, err } diff --git a/src/pkg/user/dao/dao.go b/src/pkg/user/dao/dao.go index e8a9ac890..67cbe0402 100644 --- a/src/pkg/user/dao/dao.go +++ b/src/pkg/user/dao/dao.go @@ -16,7 +16,6 @@ package dao import ( "context" - "strings" "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/lib/q" @@ -39,34 +38,19 @@ type dao struct{} // List list users func (d *dao) List(ctx context.Context, query *q.Query) ([]*models.User, error) { query = q.MustClone(query) - if query.Sorting == "" { - query.Sorting = "username" - } - - excludeAdmin := true - for key := range query.Keywords { - str := strings.ToLower(key) - if str == "user_id__in" { - excludeAdmin = false - break - } else if str == "user_id" { - excludeAdmin = false - break - } - } - - if excludeAdmin { - // Exclude admin account when not filter by UserIDs, see https://github.com/goharbor/harbor/issues/2527 - query.Keywords["user_id__gt"] = 1 - } + query.Keywords["deleted"] = false qs, err := orm.QuerySetter(ctx, &models.User{}, query) if err != nil { return nil, err } - users := []*models.User{} - if _, err := qs.OrderBy(query.Sorting).All(&users); err != nil { + if query.Sorting != "" { + qs = qs.OrderBy(query.Sorting) + } + + var users []*models.User + if _, err := qs.All(&users); err != nil { return nil, err } diff --git a/src/pkg/user/dao/dao_test.go b/src/pkg/user/dao/dao_test.go index f61ffc89d..87d7f0275 100644 --- a/src/pkg/user/dao/dao_test.go +++ b/src/pkg/user/dao/dao_test.go @@ -39,6 +39,12 @@ func (suite *DaoTestSuite) TestList() { suite.Nil(err) suite.Len(users, 1) } + + { + users, err := suite.dao.List(orm.Context(), q.New(q.KeyWords{"username": "admin"})) + suite.Nil(err) + suite.Len(users, 1) + } } func TestDaoTestSuite(t *testing.T) { diff --git a/src/pkg/user/manager.go b/src/pkg/user/manager.go index 25a6695f8..ad3d08509 100644 --- a/src/pkg/user/manager.go +++ b/src/pkg/user/manager.go @@ -16,7 +16,9 @@ package user import ( "context" + "strings" + "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/user/dao" "github.com/goharbor/harbor/src/pkg/user/models" @@ -29,6 +31,10 @@ var ( // Manager is used for user management type Manager interface { + // Get get user by user id + Get(ctx context.Context, id int) (*models.User, error) + // Get get user by username + GetByName(ctx context.Context, username string) (*models.User, error) // List users according to the query List(ctx context.Context, query *q.Query) (models.Users, error) } @@ -42,7 +48,57 @@ type manager struct { dao dao.DAO } +// Get get user by user id +func (m *manager) Get(ctx context.Context, id int) (*models.User, error) { + users, err := m.dao.List(ctx, q.New(q.KeyWords{"user_id": id})) + if err != nil { + return nil, err + } + + if len(users) == 0 { + return nil, errors.NotFoundError(nil).WithMessage("user %d not found", id) + } + + return users[0], nil +} + +// Get get user by username +func (m *manager) GetByName(ctx context.Context, username string) (*models.User, error) { + users, err := m.dao.List(ctx, q.New(q.KeyWords{"username": username})) + if err != nil { + return nil, err + } + + if len(users) == 0 { + return nil, errors.NotFoundError(nil).WithMessage("user %s not found", username) + } + + return users[0], nil +} + // List users according to the query func (m *manager) List(ctx context.Context, query *q.Query) (models.Users, error) { + query = q.MustClone(query) + if query.Sorting == "" { + query.Sorting = "username" + } + + excludeAdmin := true + for key := range query.Keywords { + str := strings.ToLower(key) + if str == "user_id__in" { + excludeAdmin = false + break + } else if str == "user_id" { + excludeAdmin = false + break + } + } + + if excludeAdmin { + // Exclude admin account when not filter by UserIDs, see https://github.com/goharbor/harbor/issues/2527 + query.Keywords["user_id__gt"] = 1 + } + return m.dao.List(ctx, query) } diff --git a/src/server/middleware/vulnerable/vulnerable.go b/src/server/middleware/vulnerable/vulnerable.go index 43526f836..d0f1278bd 100644 --- a/src/server/middleware/vulnerable/vulnerable.go +++ b/src/server/middleware/vulnerable/vulnerable.go @@ -59,7 +59,7 @@ func Middleware() func(http.Handler) http.Handler { return err } - proj, err := projectController.Get(ctx, art.ProjectID, project.CVEAllowlist(true)) + proj, err := projectController.Get(ctx, art.ProjectID, project.WithEffectCVEAllowlist()) if err != nil { logger.Errorf("get the project %d failed, error: %v", art.ProjectID, err) return err diff --git a/src/server/v2.0/handler/base.go b/src/server/v2.0/handler/base.go index d9a70412b..b3fe1ced0 100644 --- a/src/server/v2.0/handler/base.go +++ b/src/server/v2.0/handler/base.go @@ -18,14 +18,15 @@ package handler import ( "context" + "net/http" + "net/url" + "strconv" + "github.com/go-openapi/runtime" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/errors" lib_http "github.com/goharbor/harbor/src/lib/http" "github.com/goharbor/harbor/src/lib/q" - "net/http" - "net/url" - "strconv" "github.com/go-openapi/runtime/middleware" "github.com/goharbor/harbor/src/common/rbac" @@ -115,6 +116,18 @@ func (b *BaseAPI) RequireSysAdmin(ctx context.Context) error { return nil } +// RequireAuthenticated checks it's authenticated according to the security context +func (b *BaseAPI) RequireAuthenticated(ctx context.Context) error { + secCtx, ok := security.FromContext(ctx) + if !ok { + return errors.UnauthorizedError(errors.New("security context not found")) + } + if !secCtx.IsAuthenticated() { + return errors.UnauthorizedError(nil) + } + return nil +} + // BuildQuery builds the query model according to the query string func (b *BaseAPI) BuildQuery(ctx context.Context, query *string, pageNumber, pageSize *int64) (*q.Query, error) { var ( diff --git a/src/server/v2.0/handler/model/project.go b/src/server/v2.0/handler/model/project.go new file mode 100644 index 000000000..8b7bda89a --- /dev/null +++ b/src/server/v2.0/handler/model/project.go @@ -0,0 +1,79 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "strings" + + "github.com/go-openapi/strfmt" + "github.com/goharbor/harbor/src/controller/project" + "github.com/goharbor/harbor/src/lib" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/scan/vuln" + "github.com/goharbor/harbor/src/server/v2.0/models" +) + +// Project model +type Project struct { + *project.Project +} + +// ToSwagger converts the project to the swagger model +func (p *Project) ToSwagger() *models.Project { + var currentUserRoleIds []int32 + for _, role := range p.RoleList { + currentUserRoleIds = append(currentUserRoleIds, int32(role)) + } + + var md *models.ProjectMetadata + if p.Metadata != nil { + var m models.ProjectMetadata + lib.JSONCopy(&m, p.Metadata) + + // Transform the severity to severity of CVSS v3.0 Ratings + if m.Severity != nil { + severity := strings.ToLower(vuln.ParseSeverityVersion3(*m.Severity).String()) + m.Severity = &severity + } + + md = &m + } + + var allowlist models.CVEAllowlist + if err := lib.JSONCopy(&allowlist, p.CVEAllowlist); err != nil { + log.Warningf("failed to copy CVEAllowlist form %T", p.CVEAllowlist) + } + + return &models.Project{ + ChartCount: int64(p.ChartCount), + CreationTime: strfmt.DateTime(p.CreationTime), + CurrentUserRoleID: int64(p.Role), + CurrentUserRoleIds: currentUserRoleIds, + CVEAllowlist: &allowlist, + Metadata: md, + Name: p.Name, + OwnerID: int32(p.OwnerID), + OwnerName: p.OwnerName, + ProjectID: int32(p.ProjectID), + RegistryID: p.RegistryID, + RepoCount: p.RepoCount, + UpdateTime: strfmt.DateTime(p.UpdateTime), + } +} + +// NewProject ... +func NewProject(p *project.Project) *Project { + return &Project{p} +} diff --git a/src/server/v2.0/handler/project.go b/src/server/v2.0/handler/project.go index 7f34aa21b..136b1d12d 100644 --- a/src/server/v2.0/handler/project.go +++ b/src/server/v2.0/handler/project.go @@ -2,33 +2,216 @@ package handler import ( "context" + "fmt" + "strconv" + "strings" + "sync" + "github.com/go-openapi/runtime/middleware" "github.com/go-openapi/strfmt" + "github.com/goharbor/harbor/src/common" + pro "github.com/goharbor/harbor/src/common/dao/project" "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/common/security" + "github.com/goharbor/harbor/src/common/security/local" "github.com/goharbor/harbor/src/controller/project" + "github.com/goharbor/harbor/src/controller/quota" + "github.com/goharbor/harbor/src/controller/repository" + "github.com/goharbor/harbor/src/core/api" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/lib" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/audit" + "github.com/goharbor/harbor/src/pkg/project/metadata" + "github.com/goharbor/harbor/src/pkg/quota/types" + "github.com/goharbor/harbor/src/pkg/retention/policy" + "github.com/goharbor/harbor/src/pkg/user" + "github.com/goharbor/harbor/src/replication" + "github.com/goharbor/harbor/src/server/v2.0/handler/model" "github.com/goharbor/harbor/src/server/v2.0/models" operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/project" ) +// for the proxy cache type project, we will create a 7 days retention policy for it by default +const defaultDaysToRetentionForProxyCacheProject = 7 + func newProjectAPI() *projectAPI { return &projectAPI{ - auditMgr: audit.Mgr, - proCtl: project.Ctl, + auditMgr: audit.Mgr, + metadataMgr: metadata.Mgr, + userMgr: user.Mgr, + repositoryCtl: repository.Ctl, + projectCtl: project.Ctl, + quotaCtl: quota.Ctl, } } type projectAPI struct { BaseAPI - auditMgr audit.Manager - proCtl project.Controller + auditMgr audit.Manager + metadataMgr metadata.Manager + userMgr user.Manager + repositoryCtl repository.Controller + projectCtl project.Controller + quotaCtl quota.Controller +} + +func (a *projectAPI) CreateProject(ctx context.Context, params operation.CreateProjectParams) middleware.Responder { + if err := a.RequireAuthenticated(ctx); err != nil { + return a.SendError(ctx, err) + } + + onlyAdmin, err := config.OnlyAdminCreateProject() + if err != nil { + return a.SendError(ctx, fmt.Errorf("failed to determine whether only admin can create projects: %v", err)) + } + + secCtx, _ := security.FromContext(ctx) + if onlyAdmin && !(secCtx.IsSysAdmin() || secCtx.IsSolutionUser()) { + log.Errorf("Only sys admin can create project") + return a.SendError(ctx, errors.ForbiddenError(nil).WithMessage("Only system admin can create project")) + } + + req := params.Project + + if req.RegistryID != nil && !secCtx.IsSysAdmin() { + // only system admin can create the proxy cache project + return a.SendError(ctx, errors.ForbiddenError(nil).WithMessage("Only system admin can create proxy cache project")) + } + + // populate storage limit + if config.QuotaPerProjectEnable() { + // the security context is not sys admin, set the StorageLimit the global StoragePerProject + if req.StorageLimit == nil || *req.StorageLimit == 0 || !secCtx.IsSysAdmin() { + setting, err := config.QuotaSetting() + if err != nil { + log.Errorf("failed to get quota setting: %v", err) + return a.SendError(ctx, fmt.Errorf("failed to get quota setting: %v", err)) + } + defaultStorageLimit := setting.StoragePerProject + req.StorageLimit = &defaultStorageLimit + } + } else { + // ignore storage limit when quota per project disabled + req.StorageLimit = nil + } + + if req.Metadata == nil { + req.Metadata = &models.ProjectMetadata{} + } + + // accept the "public" property to make replication work well with old versions(<=1.2.0) + if req.Public != nil && req.Metadata.Public == "" { + req.Metadata.Public = strconv.FormatBool(*req.Public) + } + + // populate public metadata as false if it isn't set + if req.Metadata.Public == "" { + req.Metadata.Public = strconv.FormatBool(false) + } + + // validate the RegistryID and StorageLimit in the body of the request + if err := a.validateProjectReq(ctx, req); err != nil { + return a.SendError(ctx, err) + } + + var ownerID int + // set the owner as the system admin when the API being called by replication + // it's a solution to workaround the restriction of project creation API: + // only normal users can create projects + if secCtx.IsSolutionUser() { + ownerID = 1 + } else { + ownerName := secCtx.GetUsername() + user, err := a.userMgr.GetByName(ctx, ownerName) + if err != nil { + return a.SendError(ctx, err) + } + ownerID = user.UserID + } + + p := &project.Project{ + Name: req.ProjectName, + OwnerID: ownerID, + RegistryID: lib.Int64Value(req.RegistryID), + } + lib.JSONCopy(&p.Metadata, req.Metadata) + + projectID, err := a.projectCtl.Create(ctx, p) + if err != nil { + return a.SendError(ctx, err) + } + + // StorageLimit is provided in the request body and it's valid, + // create the quota for the project + if req.StorageLimit != nil { + referenceID := quota.ReferenceID(projectID) + hardLimits := types.ResourceList{types.ResourceStorage: *req.StorageLimit} + if _, err := a.quotaCtl.Create(ctx, quota.ProjectReference, referenceID, hardLimits); err != nil { + return a.SendError(ctx, fmt.Errorf("failed to create quota for project: %v", err)) + } + } + + // RegistryID is provided in the request body and it's valid, + // create a default retention policy for proxy project + if req.RegistryID != nil { + plc := policy.WithNDaysSinceLastPull(projectID, defaultDaysToRetentionForProxyCacheProject) + // TODO: move the retention controller to `src/controller/retention` and + // change to use the default retention controller in `src/controller/retention` + retentionID, err := api.GetRetentionController().CreateRetention(plc) + if err != nil { + return a.SendError(ctx, err) + } + md := map[string]string{"retention_id": strconv.FormatInt(retentionID, 10)} + if err := a.metadataMgr.Add(ctx, projectID, md); err != nil { + return a.SendError(ctx, err) + } + return nil + } + + location := fmt.Sprintf("%s/%d", strings.TrimSuffix(params.HTTPRequest.URL.Path, "/"), projectID) + return operation.NewCreateProjectCreated().WithLocation(location) +} + +func (a *projectAPI) DeleteProject(ctx context.Context, params operation.DeleteProjectParams) middleware.Responder { + if err := a.RequireProjectAccess(ctx, params.ProjectID, rbac.ActionDelete); err != nil { + return a.SendError(ctx, err) + } + + result, err := a.deletable(ctx, params.ProjectID) + if err != nil { + return a.SendError(ctx, err) + } + + if !result.Deletable { + return a.SendError(ctx, errors.PreconditionFailedError(errors.New(result.Message))) + } + + if err := a.projectCtl.Delete(ctx, params.ProjectID); err != nil { + return a.SendError(ctx, err) + } + + referenceID := quota.ReferenceID(params.ProjectID) + q, err := a.quotaCtl.GetByRef(ctx, quota.ProjectReference, referenceID) + if err != nil { + log.Warningf("failed to get quota for project %d, error: %v", params.ProjectID, err) + } else { + if err := a.quotaCtl.Delete(ctx, q.ID); err != nil { + return a.SendError(ctx, fmt.Errorf("failed to delete quota for project: %v", err)) + } + } + + return operation.NewDeleteProjectOK() } func (a *projectAPI) GetLogs(ctx context.Context, params operation.GetLogsParams) middleware.Responder { if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionList, rbac.ResourceLog); err != nil { return a.SendError(ctx, err) } - pro, err := a.proCtl.GetByName(ctx, params.ProjectName) + pro, err := a.projectCtl.GetByName(ctx, params.ProjectName) if err != nil { return a.SendError(ctx, err) } @@ -63,3 +246,393 @@ func (a *projectAPI) GetLogs(ctx context.Context, params operation.GetLogsParams WithLink(a.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()). WithPayload(auditLogs) } + +func (a *projectAPI) GetProject(ctx context.Context, params operation.GetProjectParams) middleware.Responder { + if err := a.RequireProjectAccess(ctx, params.ProjectID, rbac.ActionRead); err != nil { + return a.SendError(ctx, err) + } + + p, err := a.getProject(ctx, params.ProjectID, project.WithCVEAllowlist(), project.WithOwner()) + if err != nil { + return a.SendError(ctx, err) + } + + return operation.NewGetProjectOK().WithPayload(model.NewProject(p).ToSwagger()) +} + +func (a *projectAPI) GetProjectDeletable(ctx context.Context, params operation.GetProjectDeletableParams) middleware.Responder { + if err := a.RequireProjectAccess(ctx, params.ProjectID, rbac.ActionDelete); err != nil { + return a.SendError(ctx, err) + } + + result, err := a.deletable(ctx, params.ProjectID) + if err != nil { + return a.SendError(ctx, err) + } + + return operation.NewGetProjectDeletableOK().WithPayload(result) +} + +func (a *projectAPI) GetProjectSummary(ctx context.Context, params operation.GetProjectSummaryParams) middleware.Responder { + if err := a.RequireProjectAccess(ctx, params.ProjectID, rbac.ActionRead); err != nil { + return a.SendError(ctx, err) + } + + p, err := a.getProject(ctx, params.ProjectID) + if err != nil { + return a.SendError(ctx, err) + } + + summary := &models.ProjectSummary{ + ChartCount: int64(p.ChartCount), + RepoCount: p.RepoCount, + } + + var fetchSummaries []func(context.Context, *project.Project, *models.ProjectSummary) + + if hasPerm := a.HasProjectPermission(ctx, p.ProjectID, rbac.ActionRead, rbac.ResourceQuota); hasPerm { + fetchSummaries = append(fetchSummaries, getProjectQuotaSummary) + } + + if hasPerm := a.HasProjectPermission(ctx, p.ProjectID, rbac.ActionList, rbac.ResourceMember); hasPerm { + fetchSummaries = append(fetchSummaries, getProjectMemberSummary) + } + + if p.RegistryID > 0 { + fetchSummaries = append(fetchSummaries, getProjectRegistrySummary) + } + + var wg sync.WaitGroup + for _, fn := range fetchSummaries { + fn := fn + + wg.Add(1) + go func() { + defer wg.Done() + fn(ctx, p, summary) + }() + } + wg.Wait() + + return operation.NewGetProjectSummaryOK().WithPayload(summary) +} + +func (a *projectAPI) HeadProject(ctx context.Context, params operation.HeadProjectParams) middleware.Responder { + if err := a.RequireAuthenticated(ctx); err != nil { + return a.SendError(ctx, err) + } + + if _, err := a.projectCtl.GetByName(ctx, params.ProjectName); err != nil { + return a.SendError(ctx, err) + } + + return operation.NewHeadProjectOK() +} + +func (a *projectAPI) ListProjects(ctx context.Context, params operation.ListProjectsParams) middleware.Responder { + query := q.New(q.KeyWords{}) + query.Sorting = "name" + query.PageNumber = *params.Page + query.PageSize = *params.PageSize + + if name := lib.StringValue(params.Name); name != "" { + query.Keywords["name"] = &q.FuzzyMatchValue{Value: name} + } + if owner := lib.StringValue(params.Owner); owner != "" { + query.Keywords["owner"] = owner + } + if params.Public != nil { + query.Keywords["public"] = lib.BoolValue(params.Public) + } + + secCtx, ok := security.FromContext(ctx) + if ok && secCtx.IsAuthenticated() { + if !secCtx.IsSysAdmin() && !secCtx.IsSolutionUser() { + // authenticated but not system admin or solution user, + // return public projects and projects that the user is member of + if l, ok := secCtx.(*local.SecurityContext); ok { + currentUser := l.User() + member := &project.MemberQuery{ + Name: currentUser.Username, + GroupIDs: currentUser.GroupIDs, + } + + // not filter by public or filter by the public with true, + // so also return public projects for the member + if public, ok := query.Keywords["public"]; !ok || lib.ToBool(public) { + member.WithPublic = true + } + + query.Keywords["member"] = member + } else { + // can't get the user info, force to return public projects + query.Keywords["public"] = true + } + } + } else { + if params.Public != nil && !*params.Public { + // anonymous want to query private projects return empty projects directly + return operation.NewListProjectsOK().WithXTotalCount(0).WithPayload([]*models.Project{}) + } + // force to return public projects for anonymous + query.Keywords["public"] = true + } + + total, err := a.projectCtl.Count(ctx, query) + if err != nil { + return a.SendError(ctx, err) + } + + if total == 0 { + // no projects found for the query return directly + return operation.NewListProjectsOK().WithXTotalCount(0).WithPayload([]*models.Project{}) + } + + projects, err := a.projectCtl.List(ctx, query, project.WithCVEAllowlist(), project.WithOwner()) + if err != nil { + return a.SendError(ctx, err) + } + + var wg sync.WaitGroup + for _, p := range projects { + wg.Add(1) + go func(p *project.Project) { + defer wg.Done() + // due to the issue https://github.com/lib/pq/issues/81 of lib/pg or postgres, + // simultaneous queries in transaction may failed, so clone a ctx with new ormer here + if err := a.populateProperties(orm.Clone(ctx), p); err != nil { + log.G(ctx).Errorf("failed to populate propertites for project %s, error: %v", p.Name, err) + } + }(p) + } + wg.Wait() + + var payload []*models.Project + for _, p := range projects { + payload = append(payload, model.NewProject(p).ToSwagger()) + } + + return operation.NewListProjectsOK(). + WithXTotalCount(total). + WithLink(a.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()). + WithPayload(payload) +} + +func (a *projectAPI) UpdateProject(ctx context.Context, params operation.UpdateProjectParams) middleware.Responder { + if err := a.RequireProjectAccess(ctx, params.ProjectID, rbac.ActionUpdate); err != nil { + return a.SendError(ctx, err) + } + + p, err := a.projectCtl.Get(ctx, params.ProjectID, project.Metadata(false)) + if err != nil { + return a.SendError(ctx, err) + } + + if params.Project.CVEAllowlist != nil { + if params.Project.CVEAllowlist.ProjectID == 0 { + // project_id in cve_allowlist not provided or provided as 0, let it to be the id of the project which will be updating + params.Project.CVEAllowlist.ProjectID = params.ProjectID + } else if params.Project.CVEAllowlist.ProjectID != params.ProjectID { + return a.SendError(ctx, errors.BadRequestError(nil). + WithMessage("project_id in cve_allowlist must be %d but it's %d", params.ProjectID, params.Project.CVEAllowlist.ProjectID)) + } + + if err := lib.JSONCopy(&p.CVEAllowlist, params.Project.CVEAllowlist); err != nil { + return a.SendError(ctx, errors.UnknownError(nil).WithMessage("failed to process cve_allowlist, error: %v", err)) + } + } + + lib.JSONCopy(&p.Metadata, params.Project.Metadata) + + if err := a.projectCtl.Update(ctx, p); err != nil { + return a.SendError(ctx, err) + } + + return operation.NewUpdateProjectOK() +} + +func (a *projectAPI) deletable(ctx context.Context, projectID int64) (*models.ProjectDeletable, error) { + proj, err := a.getProject(ctx, projectID) + if err != nil { + return nil, err + } + + result := &models.ProjectDeletable{Deletable: true} + if proj.RepoCount > 0 { + result.Deletable = false + result.Message = "the project contains repositories, can not be deleted" + } else if proj.ChartCount > 0 { + result.Deletable = false + result.Message = "the project contains helm charts, can not be deleted" + } + + return result, nil +} + +func (a *projectAPI) getProject(ctx context.Context, projectID int64, options ...project.Option) (*project.Project, error) { + p, err := a.projectCtl.Get(ctx, projectID, options...) + if err != nil { + return nil, err + } + + if err := a.populateProperties(ctx, p); err != nil { + return nil, err + } + + return p, nil +} + +func (a *projectAPI) validateProjectReq(ctx context.Context, req *models.ProjectReq) error { + if req.RegistryID != nil { + if *req.RegistryID <= 0 { + return errors.BadRequestError(fmt.Errorf("%d is invalid value of registry_id, it should be geater than 0", *req.RegistryID)) + } + + registry, err := replication.RegistryMgr.Get(*req.RegistryID) + if err != nil { + return fmt.Errorf("failed to get the registry %d: %v", *req.RegistryID, err) + } + if registry == nil { + return errors.NotFoundError(fmt.Errorf("registry %d not found", *req.RegistryID)) + } + permitted := false + for _, t := range config.GetPermittedRegistryTypesForProxyCache() { + if string(registry.Type) == t { + permitted = true + break + } + } + if !permitted { + return errors.BadRequestError(fmt.Errorf("unsupported registry type %s", string(registry.Type))) + } + } + + if req.StorageLimit != nil { + hardLimits := types.ResourceList{types.ResourceStorage: *req.StorageLimit} + if err := quota.Validate(ctx, quota.ProjectReference, hardLimits); err != nil { + return errors.BadRequestError(err) + } + } + + return nil +} + +func (a *projectAPI) populateProperties(ctx context.Context, p *project.Project) error { + if secCtx, ok := security.FromContext(ctx); ok { + if sc, ok := secCtx.(*local.SecurityContext); ok { + roles, err := pro.ListRoles(sc.User(), p.ProjectID) + if err != nil { + return err + } + p.RoleList = roles + p.Role = highestRole(roles) + } + } + + total, err := a.repositoryCtl.Count(ctx, q.New(q.KeyWords{"project_id": p.ProjectID})) + if err != nil { + return err + } + p.RepoCount = total + + // Populate chart count property + if config.WithChartMuseum() { + count, err := api.GetChartController().GetCountOfCharts([]string{p.Name}) + if err != nil { + err = errors.Wrap(err, fmt.Sprintf("get chart count of project %d failed", p.ProjectID)) + return err + } + + p.ChartCount = count + } + return nil +} + +func getProjectQuotaSummary(ctx context.Context, p *project.Project, summary *models.ProjectSummary) { + if !config.QuotaPerProjectEnable() { + log.Debug("Quota per project disabled") + return + } + + q, err := quota.Ctl.GetByRef(ctx, quota.ProjectReference, quota.ReferenceID(p.ProjectID)) + if err != nil { + log.Warningf("failed to get quota for project: %d", p.ProjectID) + return + } + + summary.Quota = &models.ProjectSummaryQuota{} + if hard, err := q.GetHard(); err == nil { + lib.JSONCopy(&summary.Quota.Hard, hard) + } + if used, err := q.GetUsed(); err == nil { + lib.JSONCopy(&summary.Quota.Used, used) + } +} + +func getProjectMemberSummary(ctx context.Context, p *project.Project, summary *models.ProjectSummary) { + var wg sync.WaitGroup + + for _, e := range []struct { + role int + count *int64 + }{ + {common.RoleProjectAdmin, &summary.ProjectAdminCount}, + {common.RoleMaintainer, &summary.MaintainerCount}, + {common.RoleDeveloper, &summary.DeveloperCount}, + {common.RoleGuest, &summary.GuestCount}, + {common.RoleLimitedGuest, &summary.LimitedGuestCount}, + } { + wg.Add(1) + go func(role int, count *int64) { + defer wg.Done() + + total, err := pro.GetTotalOfProjectMembers(p.ProjectID, role) + if err != nil { + log.Warningf("failed to get total of project members of role %d", role) + return + } + + *count = total + }(e.role, e.count) + } + + wg.Wait() +} + +func getProjectRegistrySummary(ctx context.Context, p *project.Project, summary *models.ProjectSummary) { + if p.RegistryID <= 0 { + return + } + + registry, err := replication.RegistryMgr.Get(p.RegistryID) + if err != nil { + log.Warningf("failed to get registry %d: %v", p.RegistryID, err) + } else if registry != nil { + registry.Credential = nil + lib.JSONCopy(&summary.Registry, registry) + } +} + +// Returns the highest role in the role list. +// This func should be removed once we deprecate the "current_user_role_id" in project API +// A user can have multiple roles and they may not have a strict ranking relationship +func highestRole(roles []int) int { + if roles == nil { + return 0 + } + rolePower := map[int]int{ + common.RoleProjectAdmin: 50, + common.RoleMaintainer: 40, + common.RoleDeveloper: 30, + common.RoleGuest: 20, + common.RoleLimitedGuest: 10, + } + var highest, highestPower int + for _, role := range roles { + if p, ok := rolePower[role]; ok && p > highestPower { + highest = role + highestPower = p + } + } + return highest +} diff --git a/src/server/v2.0/route/legacy.go b/src/server/v2.0/route/legacy.go index 31074a41e..9e3b56bb1 100755 --- a/src/server/v2.0/route/legacy.go +++ b/src/server/v2.0/route/legacy.go @@ -24,8 +24,6 @@ import ( func registerLegacyRoutes() { version := APIVersion beego.Router("/api/"+version+"/projects/:pid([0-9]+)/members/?:pmid([0-9]+)", &api.ProjectMemberAPI{}) - beego.Router("/api/"+version+"/projects/", &api.ProjectAPI{}, "head:Head") - beego.Router("/api/"+version+"/projects/:id([0-9]+)", &api.ProjectAPI{}) beego.Router("/api/"+version+"/users/:id", &api.UserAPI{}, "get:Get;delete:Delete;put:Put") beego.Router("/api/"+version+"/users", &api.UserAPI{}, "get:List;post:Post") beego.Router("/api/"+version+"/users/search", &api.UserAPI{}, "get:Search") @@ -42,9 +40,6 @@ func registerLegacyRoutes() { beego.Router("/api/"+version+"/health", &api.HealthAPI{}, "get:CheckHealth") beego.Router("/api/"+version+"/ping", &api.SystemInfoAPI{}, "get:Ping") beego.Router("/api/"+version+"/search", &api.SearchAPI{}) - beego.Router("/api/"+version+"/projects/", &api.ProjectAPI{}, "get:List;post:Post") - beego.Router("/api/"+version+"/projects/:id([0-9]+)/summary", &api.ProjectAPI{}, "get:Summary") - beego.Router("/api/"+version+"/projects/:id([0-9]+)/_deletable", &api.ProjectAPI{}, "get:Deletable") beego.Router("/api/"+version+"/projects/:id([0-9]+)/metadatas/?:name", &api.MetadataAPI{}, "get:Get") beego.Router("/api/"+version+"/projects/:id([0-9]+)/metadatas/", &api.MetadataAPI{}, "post:Post") beego.Router("/api/"+version+"/projects/:pid([0-9]+)/robots", &api.RobotAPI{}, "post:Post;get:List") diff --git a/src/testing/controller/project/controller.go b/src/testing/controller/project/controller.go index d0a468dcc..19889b64b 100644 --- a/src/testing/controller/project/controller.go +++ b/src/testing/controller/project/controller.go @@ -135,7 +135,7 @@ func (_m *Controller) GetByName(ctx context.Context, projectName string, options } // List provides a mock function with given fields: ctx, query, options -func (_m *Controller) List(ctx context.Context, query *models.ProjectQueryParam, options ...project.Option) ([]*models.Project, error) { +func (_m *Controller) List(ctx context.Context, query *q.Query, options ...project.Option) ([]*models.Project, error) { _va := make([]interface{}, len(options)) for _i := range options { _va[_i] = options[_i] @@ -146,7 +146,7 @@ func (_m *Controller) List(ctx context.Context, query *models.ProjectQueryParam, ret := _m.Called(_ca...) var r0 []*models.Project - if rf, ok := ret.Get(0).(func(context.Context, *models.ProjectQueryParam, ...project.Option) []*models.Project); ok { + if rf, ok := ret.Get(0).(func(context.Context, *q.Query, ...project.Option) []*models.Project); ok { r0 = rf(ctx, query, options...) } else { if ret.Get(0) != nil { @@ -155,7 +155,7 @@ func (_m *Controller) List(ctx context.Context, query *models.ProjectQueryParam, } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *models.ProjectQueryParam, ...project.Option) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *q.Query, ...project.Option) error); ok { r1 = rf(ctx, query, options...) } else { r1 = ret.Error(1) @@ -163,3 +163,17 @@ func (_m *Controller) List(ctx context.Context, query *models.ProjectQueryParam, return r0, r1 } + +// Update provides a mock function with given fields: ctx, _a1 +func (_m *Controller) Update(ctx context.Context, _a1 *models.Project) error { + ret := _m.Called(ctx, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *models.Project) error); ok { + r0 = rf(ctx, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/src/testing/pkg/project/manager.go b/src/testing/pkg/project/manager.go index 14ef4dd50..5444509c6 100644 --- a/src/testing/pkg/project/manager.go +++ b/src/testing/pkg/project/manager.go @@ -96,19 +96,12 @@ func (_m *Manager) Get(ctx context.Context, idOrName interface{}) (*models.Proje } // List provides a mock function with given fields: ctx, query -func (_m *Manager) List(ctx context.Context, query ...*models.ProjectQueryParam) ([]*models.Project, error) { - _va := make([]interface{}, len(query)) - for _i := range query { - _va[_i] = query[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) +func (_m *Manager) List(ctx context.Context, query *q.Query) ([]*models.Project, error) { + ret := _m.Called(ctx, query) var r0 []*models.Project - if rf, ok := ret.Get(0).(func(context.Context, ...*models.ProjectQueryParam) []*models.Project); ok { - r0 = rf(ctx, query...) + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*models.Project); ok { + r0 = rf(ctx, query) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*models.Project) @@ -116,8 +109,8 @@ func (_m *Manager) List(ctx context.Context, query ...*models.ProjectQueryParam) } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, ...*models.ProjectQueryParam) error); ok { - r1 = rf(ctx, query...) + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) } else { r1 = ret.Error(1) } diff --git a/src/testing/pkg/user/manager.go b/src/testing/pkg/user/manager.go index 141a182a2..8833b1e62 100644 --- a/src/testing/pkg/user/manager.go +++ b/src/testing/pkg/user/manager.go @@ -5,10 +5,12 @@ package user import ( context "context" - models "github.com/goharbor/harbor/src/pkg/user/models" + models "github.com/goharbor/harbor/src/common/models" mock "github.com/stretchr/testify/mock" q "github.com/goharbor/harbor/src/lib/q" + + usermodels "github.com/goharbor/harbor/src/pkg/user/models" ) // Manager is an autogenerated mock type for the Manager type @@ -16,16 +18,62 @@ type Manager struct { mock.Mock } +// Get provides a mock function with given fields: ctx, id +func (_m *Manager) Get(ctx context.Context, id int) (*models.User, error) { + ret := _m.Called(ctx, id) + + var r0 *models.User + if rf, ok := ret.Get(0).(func(context.Context, int) *models.User); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*models.User) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetByName provides a mock function with given fields: ctx, username +func (_m *Manager) GetByName(ctx context.Context, username string) (*models.User, error) { + ret := _m.Called(ctx, username) + + var r0 *models.User + if rf, ok := ret.Get(0).(func(context.Context, string) *models.User); ok { + r0 = rf(ctx, username) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*models.User) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, username) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // List provides a mock function with given fields: ctx, query -func (_m *Manager) List(ctx context.Context, query *q.Query) (models.Users, error) { +func (_m *Manager) List(ctx context.Context, query *q.Query) (usermodels.Users, error) { ret := _m.Called(ctx, query) - var r0 models.Users - if rf, ok := ret.Get(0).(func(context.Context, *q.Query) models.Users); ok { + var r0 usermodels.Users + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) usermodels.Users); ok { r0 = rf(ctx, query) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(models.Users) + r0 = ret.Get(0).(usermodels.Users) } } diff --git a/tests/apitests/python/library/base.py b/tests/apitests/python/library/base.py index 44a96e399..a2c9a83c6 100644 --- a/tests/apitests/python/library/base.py +++ b/tests/apitests/python/library/base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import os import sys import time import subprocess @@ -22,6 +22,10 @@ class Credential: self.username = username self.password = password +def get_endpoint(): + harbor_server = os.environ.get("HARBOR_HOST", "localhost:8080") + return os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://"+harbor_server+"/api/v2.0" + def _create_client(server, credential, debug, api_type="products"): cfg = None if api_type in ('projectv2', 'artifact', 'repository', 'scan'): @@ -85,13 +89,16 @@ def run_command(command): raise Exception('Error: Exited with error code: %s. Output:%s'% (e.returncode, e.output)) return output -class Base: - def __init__(self, - server = Server(endpoint="http://localhost:8080/api", verify_ssl=False), - credential = Credential(type="basic_auth", username="admin", password="Harbor12345"), - debug = True, api_type = "products"): +class Base(object): + def __init__(self, server=None, credential=None, debug=True, api_type="products"): + if server is None: + server = Server(endpoint=get_endpoint(), verify_ssl=False) if not isinstance(server.verify_ssl, bool): server.verify_ssl = server.verify_ssl == "True" + + if credential is None: + credential = Credential(type="basic_auth", username="admin", password="Harbor12345") + self.server = server self.credential = credential self.debug = debug @@ -102,8 +109,6 @@ class Base: if len(kwargs) == 0: return self.client server = self.server - if "api_type" in kwargs: - server.api_type = kwargs.get("api_type") if "endpoint" in kwargs: server.endpoint = kwargs.get("endpoint") if "verify_ssl" in kwargs: @@ -118,4 +123,4 @@ class Base: credential.username = kwargs.get("username") if "password" in kwargs: credential.password = kwargs.get("password") - return _create_client(server, credential, self.debug, self.api_type) \ No newline at end of file + return _create_client(server, credential, self.debug, kwargs.get('api_type', self.api_type)) diff --git a/tests/apitests/python/library/project.py b/tests/apitests/python/library/project.py index 0d7cf5fd6..d51b6a11a 100644 --- a/tests/apitests/python/library/project.py +++ b/tests/apitests/python/library/project.py @@ -2,7 +2,8 @@ import base import swagger_client -from swagger_client.rest import ApiException +import v2_swagger_client +from v2_swagger_client.rest import ApiException def is_member_exist_in_project(members, member_user_name, expected_member_role_id = None): result = False @@ -22,6 +23,12 @@ def get_member_id_by_name(members, member_user_name): return None class Project(base.Base): + def __init__(self, username=None, password=None): + kwargs = dict(api_type="projectv2") + if username and password: + kwargs["credential"] = base.Credential('basic_auth', username, password) + super(Project, self).__init__(**kwargs) + def create_project(self, name=None, metadata=None, expect_status_code = 201, expect_response_body = None, **kwargs): if name is None: name = base._random_name("project") @@ -30,7 +37,7 @@ class Project(base.Base): client = self._get_client(**kwargs) try: - _, status_code, header = client.projects_post_with_http_info(swagger_client.ProjectReq(name, metadata)) + _, status_code, header = client.create_project_with_http_info(v2_swagger_client.ProjectReq(project_name=name, metadata=metadata)) except ApiException as e: base._assert_status_code(expect_status_code, e.status) if expect_response_body is not None: @@ -43,7 +50,7 @@ class Project(base.Base): def get_projects(self, params, **kwargs): client = self._get_client(**kwargs) data = [] - data, status_code, _ = client.projects_get_with_http_info(**params) + data, status_code, _ = client.list_projects_with_http_info(**params) base._assert_status_code(200, status_code) return data @@ -66,7 +73,7 @@ class Project(base.Base): def check_project_name_exist(self, name=None, **kwargs): client = self._get_client(**kwargs) try: - _, status_code, _ = client.projects_head_with_http_info(name) + _, status_code, _ = client.head_project_with_http_info(name) except ApiException as e: status_code = -1 return { @@ -77,7 +84,7 @@ class Project(base.Base): def get_project(self, project_id, expect_status_code = 200, expect_response_body = None, **kwargs): client = self._get_client(**kwargs) try: - data, status_code, _ = client.projects_project_id_get_with_http_info(project_id) + data, status_code, _ = client.get_project_with_http_info(project_id) except ApiException as e: base._assert_status_code(expect_status_code, e.status) if expect_response_body is not None: @@ -90,9 +97,9 @@ class Project(base.Base): def update_project(self, project_id, expect_status_code=200, metadata=None, cve_allowlist=None, **kwargs): client = self._get_client(**kwargs) - project = swagger_client.ProjectReq(metadata=metadata, cve_allowlist=cve_allowlist) + project = v2_swagger_client.ProjectReq(metadata=metadata, cve_allowlist=cve_allowlist) try: - _, sc, _ = client.projects_project_id_put_with_http_info(project_id, project) + _, sc, _ = client.update_project_with_http_info(project_id, project) except ApiException as e: base._assert_status_code(expect_status_code, e.status) else: @@ -100,31 +107,34 @@ class Project(base.Base): def delete_project(self, project_id, expect_status_code = 200, **kwargs): client = self._get_client(**kwargs) - _, status_code, _ = client.projects_project_id_delete_with_http_info(project_id) + _, status_code, _ = client.delete_project_with_http_info(project_id) base._assert_status_code(expect_status_code, status_code) - def get_project_log(self, project_id, expect_status_code = 200, **kwargs): + def get_project_log(self, project_name, expect_status_code = 200, **kwargs): client = self._get_client(**kwargs) - body, status_code, _ = client.projects_project_id_logs_get_with_http_info(project_id) + body, status_code, _ = client.get_logs_with_http_info(project_name) base._assert_status_code(expect_status_code, status_code) return body - def filter_project_logs(self, project_id, operator, repository, tag, operation_type, **kwargs): - access_logs = self.get_project_log(project_id, **kwargs) + def filter_project_logs(self, project_name, operator, resource, resource_type, operation, **kwargs): + access_logs = self.get_project_log(project_name, **kwargs) count = 0 for each_access_log in list(access_logs): if each_access_log.username == operator and \ - each_access_log.repo_name.strip(r'/') == repository and \ - each_access_log.repo_tag == tag and \ - each_access_log.operation == operation_type: + each_access_log.resource_type == resource_type and \ + each_access_log.resource == resource and \ + each_access_log.operation == operation: count = count + 1 return count def get_project_members(self, project_id, **kwargs): + kwargs['api_type'] = 'products' client = self._get_client(**kwargs) return client.projects_project_id_members_get(project_id) def get_project_member(self, project_id, member_id, expect_status_code = 200, expect_response_body = None, **kwargs): + from swagger_client.rest import ApiException + kwargs['api_type'] = 'products' client = self._get_client(**kwargs) data = [] try: @@ -140,6 +150,7 @@ class Project(base.Base): return data def get_project_member_id(self, project_id, member_user_name, **kwargs): + kwargs['api_type'] = 'products' members = self.get_project_members(project_id, **kwargs) result = get_member_id_by_name(list(members), member_user_name) if result == None: @@ -148,18 +159,21 @@ class Project(base.Base): return result def check_project_member_not_exist(self, project_id, member_user_name, **kwargs): + kwargs['api_type'] = 'products' members = self.get_project_members(project_id, **kwargs) result = is_member_exist_in_project(list(members), member_user_name) if result == True: raise Exception(r"User {} should not be a member of project with ID {}.".format(member_user_name, project_id)) def check_project_members_exist(self, project_id, member_user_name, expected_member_role_id = None, **kwargs): + kwargs['api_type'] = 'products' members = self.get_project_members(project_id, **kwargs) result = is_member_exist_in_project(members, member_user_name, expected_member_role_id = expected_member_role_id) if result == False: raise Exception(r"User {} should be a member of project with ID {}.".format(member_user_name, project_id)) def update_project_member_role(self, project_id, member_id, member_role_id, expect_status_code = 200, **kwargs): + kwargs['api_type'] = 'products' client = self._get_client(**kwargs) role = swagger_client.Role(role_id = member_role_id) data, status_code, _ = client.projects_project_id_members_mid_put_with_http_info(project_id, member_id, role = role) @@ -168,12 +182,14 @@ class Project(base.Base): return data def delete_project_member(self, project_id, member_id, expect_status_code = 200, **kwargs): + kwargs['api_type'] = 'products' client = self._get_client(**kwargs) _, status_code, _ = client.projects_project_id_members_mid_delete_with_http_info(project_id, member_id) base._assert_status_code(expect_status_code, status_code) base._assert_status_code(200, status_code) def add_project_members(self, project_id, user_id, member_role_id = None, expect_status_code = 201, **kwargs): + kwargs['api_type'] = 'products' if member_role_id is None: member_role_id = 1 _member_user = {"user_id": int(user_id)} @@ -185,6 +201,7 @@ class Project(base.Base): return base._get_id_from_header(header) def add_project_robot_account(self, project_id, project_name, expires_at, robot_name = None, robot_desc = None, has_pull_right = True, has_push_right = True, has_chart_read_right = True, has_chart_create_right = True, expect_status_code = 201, **kwargs): + kwargs['api_type'] = 'products' if robot_name is None: robot_name = base._random_name("robot") if robot_desc is None: @@ -221,11 +238,13 @@ class Project(base.Base): return base._get_id_from_header(header), data def get_project_robot_account_by_id(self, project_id, robot_id, **kwargs): + kwargs['api_type'] = 'products' client = self._get_client(**kwargs) data, status_code, _ = client.projects_project_id_robots_robot_id_get_with_http_info(project_id, robot_id) return data def disable_project_robot_account(self, project_id, robot_id, disable, expect_status_code = 200, **kwargs): + kwargs['api_type'] = 'products' client = self._get_client(**kwargs) robotAccountUpdate = swagger_client.RobotAccountUpdate(disable) _, status_code, _ = client.projects_project_id_robots_robot_id_put_with_http_info(project_id, robot_id, robotAccountUpdate) @@ -233,6 +252,7 @@ class Project(base.Base): base._assert_status_code(200, status_code) def delete_project_robot_account(self, project_id, robot_id, expect_status_code = 200, **kwargs): + kwargs['api_type'] = 'products' client = self._get_client(**kwargs) _, status_code, _ = client.projects_project_id_robots_robot_id_delete_with_http_info(project_id, robot_id) base._assert_status_code(expect_status_code, status_code) diff --git a/tests/apitests/python/library/repository.py b/tests/apitests/python/library/repository.py index 1b7d41e94..e3190558b 100644 --- a/tests/apitests/python/library/repository.py +++ b/tests/apitests/python/library/repository.py @@ -14,7 +14,7 @@ def pull_harbor_image(registry, username, password, image, tag, expected_login_e time.sleep(2) ret = _docker_api.docker_image_pull(r'{}/{}'.format(registry, image), tag = tag, expected_error_message = expected_error_message) -def push_image_to_project(project_name, registry, username, password, image, tag, expected_login_error_message = None, expected_error_message = None, profix_for_image = None): +def push_image_to_project(project_name, registry, username, password, image, tag, expected_login_error_message = None, expected_error_message = None, profix_for_image = None, new_image=None): _docker_api = DockerAPI() _docker_api.docker_login(registry, username, password, expected_error_message = expected_login_error_message) time.sleep(2) @@ -23,6 +23,8 @@ def push_image_to_project(project_name, registry, username, password, image, tag _docker_api.docker_image_pull(image, tag = tag) time.sleep(2) + image = new_image or image + if profix_for_image == None: new_harbor_registry, new_tag = _docker_api.docker_image_tag(r'{}:{}'.format(image, tag), r'{}/{}/{}'.format(registry, project_name, image)) else: diff --git a/tests/apitests/python/test_add_member_to_private_project.py b/tests/apitests/python/test_add_member_to_private_project.py index eb42a2c69..59e79c78d 100644 --- a/tests/apitests/python/test_add_member_to_private_project.py +++ b/tests/apitests/python/test_add_member_to_private_project.py @@ -44,7 +44,7 @@ class TestProjects(unittest.TestCase): project_001_data = self.project.get_projects(dict(public=False), **USER_001_CLIENT) #3.2 Check user-001 has no any private project - self.assertEqual(project_001_data, None, msg="user-001 should has no any private project, but we got {}".format(project_001_data)) + self.assertEqual(len(project_001_data), 0, msg="user-001 should has no any private project, but we got {}".format(project_001_data)) #4. Add user-001 as a member of project-001 result = self.project.add_project_members(project_001_id, user_001_id, **ADMIN_CLIENT) diff --git a/tests/apitests/python/test_assign_role_to_ldap_group.py b/tests/apitests/python/test_assign_role_to_ldap_group.py index 3ace8ce25..7dcdff979 100644 --- a/tests/apitests/python/test_assign_role_to_ldap_group.py +++ b/tests/apitests/python/test_assign_role_to_ldap_group.py @@ -20,20 +20,13 @@ import unittest import testutils import docker -import swagger_client - from testutils import ADMIN_CLIENT -from swagger_client.models.project import Project -from swagger_client.models.project_req import ProjectReq -from swagger_client.models.project_metadata import ProjectMetadata from swagger_client.models.project_member import ProjectMember from swagger_client.models.user_group import UserGroup from swagger_client.models.configurations import Configurations -from library.projectV2 import ProjectV2 +from library.project import Project from library.base import _assert_status_code from library.base import _random_name - - from v2_swagger_client.rest import ApiException from pprint import pprint @@ -46,23 +39,20 @@ class TestAssignRoleToLdapGroup(unittest.TestCase): repository_api = testutils.GetRepositoryApi("admin", "Harbor12345") project_id = 0 docker_client = docker.from_env() - _project_name = _random_name("test_private") + _project_name = _random_name("test-ldap-group") def setUp(self): - self.projectv2= ProjectV2() + self.project = Project() #login with admin, create a project and assign role to ldap group result = self.product_api.configurations_put(configurations=Configurations(ldap_filter="", ldap_group_attribute_name="cn", ldap_group_base_dn="ou=groups,dc=example,dc=com", ldap_group_search_filter="objectclass=groupOfNames", ldap_group_search_scope=2)) pprint(result) cfgs = self.product_api.configurations_get() pprint(cfgs) - req = ProjectReq() - req.project_name = self._project_name - req.metadata = ProjectMetadata(public="false") - result = self.product_api.projects_post(req) + result = self.project.create_project(self._project_name, dict(public="false")) pprint(result) - projs = self.product_api.projects_get(name = self._project_name) + projs = self.project.get_projects(dict(name = self._project_name)) if len(projs)>0 : project = projs[0] self.project_id = project.project_id @@ -91,33 +81,30 @@ class TestAssignRoleToLdapGroup(unittest.TestCase): result = self.product_api.projects_project_id_members_post( project_id=self.project_id, project_member=projectmember ) pprint(result) - pass def tearDown(self): - #delete images in project - result = self.repository_api.delete_repository(self._project_name, "busybox") - pprint(result) - result = self.repository_api.delete_repository(self._project_name, "busyboxdev") - pprint(result) if self.project_id > 0 : - self.product_api.projects_project_id_delete(self.project_id) - pass + # delete images in project + result = self.repository_api.delete_repository(self._project_name, "busybox") + pprint(result) + result = self.repository_api.delete_repository(self._project_name, "busyboxdev") + pprint(result) + self.project.delete_project(self.project_id) def testAssignRoleToLdapGroup(self): """Test AssignRoleToLdapGroup""" - admin_product_api = testutils.GetProductApi(username="admin_user", password="zhu88jie") - projects = admin_product_api.projects_get(name=self._project_name) + admin_product_api = Project("admin_user", "zhu88jie") + projects = admin_product_api.get_projects(dict(name=self._project_name)) self.assertTrue(len(projects) == 1) self.assertEqual(1, projects[0].current_user_role_id) - - dev_product_api = testutils.GetProductApi("dev_user", "zhu88jie") - projects = dev_product_api.projects_get(name=self._project_name) + dev_product_api = Project("dev_user", "zhu88jie") + projects = dev_product_api.get_projects(dict(name=self._project_name)) self.assertTrue(len(projects) == 1) self.assertEqual(2, projects[0].current_user_role_id) - guest_product_api = testutils.GetProductApi("guest_user", "zhu88jie") - projects = guest_product_api.projects_get(name=self._project_name) + guest_product_api = Project("guest_user", "zhu88jie") + projects = guest_product_api.get_projects(dict(name=self._project_name)) self.assertTrue(len(projects) == 1) self.assertEqual(3, projects[0].current_user_role_id) @@ -130,8 +117,6 @@ class TestAssignRoleToLdapGroup(unittest.TestCase): self.assertTrue(self.queryUserLogs(username="guest_user", password="zhu88jie")>0, "guest user can see logs") self.assertTrue(self.queryUserLogs(username="test", password="123456", status_code=403)==0, "test user can not see any logs") - pass - # admin user can push, pull images def dockerCmdLoginAdmin(self, username, password): pprint(self.docker_client.info()) @@ -143,7 +128,7 @@ class TestAssignRoleToLdapGroup(unittest.TestCase): if output.find("error")>0 : self.fail("Should not fail to push image for admin_user") self.docker_client.images.pull(repository=self.harbor_host+"/"+self._project_name+"/busybox", tag="latest") - pass + # dev user can push, pull images def dockerCmdLoginDev(self, username, password, harbor_server=harbor_host): self.docker_client.login(username=username, password=password, registry=self.harbor_host) @@ -153,7 +138,7 @@ class TestAssignRoleToLdapGroup(unittest.TestCase): output = self.docker_client.images.push(repository=self.harbor_host+"/"+self._project_name+"/busyboxdev", tag="latest") if output.find("error") >0 : self.fail("Should not fail to push images for dev_user") - pass + # guest user can pull images def dockerCmdLoginGuest(self, username, password, harbor_server=harbor_host): self.docker_client.login(username=username, password=password, registry=self.harbor_host) @@ -164,12 +149,12 @@ class TestAssignRoleToLdapGroup(unittest.TestCase): if output.find("error")<0 : self.fail("Should failed to push image for guest user") self.docker_client.images.pull(repository=self.harbor_host+"/"+self._project_name+"/busybox", tag="latest") - pass + # check can see his log in current project def queryUserLogs(self, username, password, status_code=200): client=dict(endpoint = ADMIN_CLIENT["endpoint"], username = username, password = password) try: - logs = self.projectv2.get_project_log(self._project_name, status_code, **client) + logs = self.project.get_project_log(self._project_name, status_code, **client) count = 0 for log in list(logs): count = count + 1 diff --git a/tests/apitests/python/test_ldap_admin_role.py b/tests/apitests/python/test_ldap_admin_role.py index 24b17c345..9a573fc35 100644 --- a/tests/apitests/python/test_ldap_admin_role.py +++ b/tests/apitests/python/test_ldap_admin_role.py @@ -20,10 +20,10 @@ sys.path.append(os.environ["SWAGGER_CLIENT_PATH"]) import unittest import testutils import swagger_client +from testutils import TEARDOWN from library.base import _random_name -from swagger_client.models.project_req import ProjectReq +from library.project import Project from swagger_client.models.configurations import Configurations -from swagger_client.rest import ApiException from pprint import pprint @@ -32,26 +32,30 @@ from pprint import pprint class TestLdapAdminRole(unittest.TestCase): """AccessLog unit test stubs""" product_api = testutils.GetProductApi("admin", "Harbor12345") - mike_product_api = testutils.GetProductApi("mike", "zhu88jie") project_id = 0 + def setUp(self): - pass + self.project= Project() + self.mike_product_api = Project("mike", "zhu88jie") def tearDown(self): + print("Case completed") + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def test_ClearData(self): if self.project_id > 0 : - self.mike_product_api.projects_project_id_delete(project_id=self.project_id) - pass + self.mike_product_api.delete_project(self.project_id) def testLdapAdminRole(self): """Test LdapAdminRole""" - _project_name = _random_name("test_private") + _project_name = _random_name("test-ldap-admin-role") result = self.product_api.configurations_put(configurations=Configurations(ldap_group_admin_dn="cn=harbor_users,ou=groups,dc=example,dc=com")) # Create a private project - result = self.product_api.projects_post(project=ProjectReq(project_name= _project_name)) + result = self.project.create_project(_project_name) # query project with ldap user mike - projects = self.mike_product_api.projects_get(name=_project_name) + projects = self.mike_product_api.get_projects(dict(name=_project_name)) print("=================", projects) self.assertTrue(len(projects) == 1) diff --git a/tests/apitests/python/test_project_level_cve_allowlist.py b/tests/apitests/python/test_project_level_cve_allowlist.py index 986d86600..6dbab8b1a 100644 --- a/tests/apitests/python/test_project_level_cve_allowlist.py +++ b/tests/apitests/python/test_project_level_cve_allowlist.py @@ -1,10 +1,10 @@ from __future__ import absolute_import import unittest -import swagger_client +import v2_swagger_client import time -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, TEARDOWN from library.project import Project from library.user import User @@ -50,6 +50,10 @@ class TestProjectCVEAllowlist(unittest.TestCase): self.member_id = int(m_id) def tearDown(self): + print("Case completed") + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def test_ClearData(self): print("Tearing down...") self.project.delete_project_member(self.project_pa_id, self.member_id, **ADMIN_CLIENT) self.project.delete_project(self.project_pa_id,**ADMIN_CLIENT) @@ -63,9 +67,9 @@ class TestProjectCVEAllowlist(unittest.TestCase): self.assertEqual(0, len(p.cve_allowlist.items)) # User(RA) updates the project CVE allowlist, verify it fails with Forbidden error. - item_list = [swagger_client.CVEAllowlistItem(cve_id="CVE-2019-12310")] + item_list = [v2_swagger_client.CVEAllowlistItem(cve_id="CVE-2019-12310")] exp = int(time.time()) + 1000 - wl = swagger_client.CVEAllowlist(expires_at=exp, items=item_list) + wl = v2_swagger_client.CVEAllowlist(expires_at=exp, items=item_list) self.project.update_project(self.project_pa_id, cve_allowlist=wl, expect_status_code=403, **self.USER_RA_CLIENT) # Admin user updates User(RA) as project admin. @@ -78,14 +82,14 @@ class TestProjectCVEAllowlist(unittest.TestCase): self.assertEqual(exp, p.cve_allowlist.expires_at) # User(RA) updates the project CVE allowlist with empty items list - wl2 = swagger_client.CVEAllowlist(items=[]) + wl2 = v2_swagger_client.CVEAllowlist(items=[]) self.project.update_project(self.project_pa_id, cve_allowlist=wl2, **self.USER_RA_CLIENT) p = self.project.get_project(self.project_pa_id, **self.USER_RA_CLIENT) self.assertEqual(0, len(p.cve_allowlist.items)) self.assertIsNone(p.cve_allowlist.expires_at) # User(RA) updates the project metadata to set "reuse_sys_cve_allowlist" to true. - meta = swagger_client.ProjectMetadata(reuse_sys_cve_allowlist="true") + meta = v2_swagger_client.ProjectMetadata(reuse_sys_cve_allowlist="true") self.project.update_project(self.project_pa_id, metadata=meta, **self.USER_RA_CLIENT) p = self.project.get_project(self.project_pa_id, **self.USER_RA_CLIENT) self.assertEqual("true", p.metadata.reuse_sys_cve_allowlist)