diff --git a/docs/swagger.yaml b/docs/swagger.yaml index ce165247a..b8e16e8ed 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -311,6 +311,34 @@ paths: description: User need to log in first. '500': description: Unexpected internal errors. + '/projects/{project_id}/summary': + get: + summary: Get summary of the project. + description: Get summary of the project. + parameters: + - name: project_id + in: path + type: integer + format: int64 + required: true + description: Relevant project ID + tags: + - Products + responses: + '200': + description: Get summary of the project successfully. + schema: + $ref: '#/definitions/ProjectSummary' + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '404': + description: Project ID does not exist. + '403': + description: User does not have permission to get summary of the project. + '500': + description: Unexpected internal errors. '/projects/{project_id}/metadatas': get: summary: Get project metadata. @@ -3547,6 +3575,113 @@ paths: description: User does not have permission to call this API. '500': description: Unexpected internal errors. + '/quotas': + get: + summary: List quotas + description: List quotas + tags: + - quota + parameters: + - name: reference + in: query + description: The reference type of quota. + required: false + type: string + - name: sort + in: query + type: string + required: false + description: | + Sort method, valid values include: + 'hard.resource_name', '-hard.resource_name', 'used.resource_name', '-used.resource_name'. + Here '-' stands for descending order, resource_name should be the real resource name of the quota. + - name: page + in: query + type: integer + format: int32 + required: false + description: 'The page nubmer, default is 1.' + - name: page_size + in: query + type: integer + format: int32 + required: false + description: 'The size of per page, default is 10, maximum is 100.' + responses: + '200': + description: Successfully retrieved the quotas. + schema: + type: array + items: + $ref: '#/definitions/Quota' + headers: + X-Total-Count: + description: The total count of access logs + type: integer + Link: + description: Link refers to the previous page and next page + type: string + '401': + description: User is not authenticated. + '403': + description: User does not have permission to call this API. + '500': + description: Unexpected internal errors. + '/quotas/{id}': + get: + summary: Get the specified quota + description: Get the specified quota + tags: + - quota + parameters: + - name: id + in: path + type: integer + required: true + description: Quota ID + responses: + '200': + description: Successfully retrieved the quota. + schema: + $ref: '#/definitions/Quota' + '401': + description: User need to log in first. + '403': + description: User does not have permission to call this API + '404': + description: Quota does not exist. + '500': + description: Unexpected internal errors. + put: + summary: Update the specified quota + description: Update hard limits of the specified quota + tags: + - quota + parameters: + - name: id + in: path + type: integer + required: true + description: Quota ID + - name: hard + in: body + required: true + description: The new hard limits for the quota + schema: + $ref: '#/definitions/QuotaUpdateReq' + responses: + '200': + description: Updated quota hard limits successfully. + '400': + description: Illegal format of quota update request. + '401': + description: User need to log in first. + '403': + description: User does not have permission to the quota. + '404': + description: Quota ID does not exist. + '500': + description: Unexpected internal errors. responses: OK: description: 'Success' @@ -3629,6 +3764,14 @@ definitions: metadata: description: The metadata of the project. $ref: '#/definitions/ProjectMetadata' + count_limit: + type: integer + format: int64 + description: The count quota of the project. + storage_limit: + type: integer + format: int64 + description: The storage quota of the project. Project: type: object properties: @@ -3691,6 +3834,36 @@ definitions: auto_scan: type: string description: 'Whether scan images automatically when pushing. The valid values are "true", "false".' + ProjectSummary: + type: object + properties: + repo_count: + type: integer + description: The number of the repositories under this project. + chart_count: + type: integer + description: The total number of charts under this project. + project_admin_count: + type: integer + description: The total number of project admin members. + master_count: + type: integer + description: The total number of master members. + developer_count: + type: integer + description: The total number of developer members. + guest_count: + type: integer + description: The total number of guest members. + quota: + type: object + properties: + hard: + $ref: "#/definitions/ResourceList" + description: The hard limits of the quota + used: + $ref: "#/definitions/ResourceList" + description: The used status of the quota Manifest: type: object properties: @@ -4346,6 +4519,9 @@ definitions: auth_mode: type: string description: 'The auth mode of current system, such as "db_auth", "ldap_auth"' + count_per_project: + type: string + description: The default count quota for the new created projects. email_from: type: string description: The sender name for Email notification. @@ -4412,6 +4588,9 @@ definitions: self_registration: type: boolean description: 'Whether the Harbor instance supports self-registration. If it''s set to false, admin need to add user to the instance.' + storage_per_project: + type: string + description: The default storage quota for the new created projects. token_expiration: type: integer description: 'The expiration time of the token for internal Registry, in minutes.' @@ -4437,6 +4616,9 @@ definitions: auth_mode: $ref: '#/definitions/StringConfigItem' description: 'The auth mode of current system, such as "db_auth", "ldap_auth"' + count_per_project: + $ref: '#/definitions/IntegerConfigItem' + description: The default count quota for the new created projects. email_from: $ref: '#/definitions/StringConfigItem' description: The sender name for Email notification. @@ -4503,6 +4685,9 @@ definitions: self_registration: $ref: '#/definitions/BoolConfigItem' description: 'Whether the Harbor instance supports self-registration. If it''s set to false, admin need to add user to the instance.' + storage_per_project: + $ref: '#/definitions/IntegerConfigItem' + description: The default storage quota for the new created projects. token_expiration: $ref: '#/definitions/IntegerConfigItem' description: 'The expiration time of the token for internal Registry, in minutes.' @@ -5166,3 +5351,38 @@ definitions: cve_id: type: string description: The ID of the CVE, such as "CVE-2019-10164" + ResourceList: + type: object + additionalProperties: + type: integer + QuotaUpdateReq: + type: object + properties: + hard: + $ref: "#/definitions/ResourceList" + description: The new hard limits for the quota + QuotaRefObject: + type: object + additionalProperties: {} + Quota: + type: object + description: The quota object + properties: + id: + type: integer + description: ID of the quota + ref: + $ref: "#/definitions/QuotaRefObject" + description: The reference object of the quota + hard: + $ref: "#/definitions/ResourceList" + description: The hard limits of the quota + used: + $ref: "#/definitions/ResourceList" + description: The used status of the quota + creation_time: + type: string + description: the creation time of the quota + update_time: + type: string + description: the update time of the quota \ No newline at end of file diff --git a/make/migrations/postgresql/0010_1.9.0_schema.up.sql b/make/migrations/postgresql/0010_1.9.0_schema.up.sql index 1a10420d6..0718b19ae 100644 --- a/make/migrations/postgresql/0010_1.9.0_schema.up.sql +++ b/make/migrations/postgresql/0010_1.9.0_schema.up.sql @@ -9,6 +9,91 @@ CREATE TABLE cve_whitelist ( UNIQUE (project_id) ); +CREATE TABLE blob ( + id SERIAL PRIMARY KEY NOT NULL, + /* + digest of config, layer, manifest + */ + digest varchar(255) NOT NULL, + content_type varchar(255) NOT NULL, + size int NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + UNIQUE (digest) +); + +CREATE TABLE artifact ( + id SERIAL PRIMARY KEY NOT NULL, + project_id int NOT NULL, + repo varchar(255) NOT NULL, + tag varchar(255) NOT NULL, + /* + digest of manifest + */ + digest varchar(255) NOT NULL, + /* + kind of artifact, image, chart, etc.. + */ + kind varchar(255) NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + pull_time timestamp, + push_time timestamp, + CONSTRAINT unique_artifact UNIQUE (project_id, repo, tag) +); + +/* add the table for relation of artifact and blob */ +CREATE TABLE artifact_blob ( + id SERIAL PRIMARY KEY NOT NULL, + digest_af varchar(255) NOT NULL, + digest_blob varchar(255) NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + CONSTRAINT unique_artifact_blob UNIQUE (digest_af, digest_blob) +); + +/* add quota table */ +CREATE TABLE quota ( + id SERIAL PRIMARY KEY NOT NULL, + reference VARCHAR(255) NOT NULL, + reference_id VARCHAR(255) NOT NULL, + hard JSONB NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP, + UNIQUE(reference, reference_id) +); + +/* add quota usage table */ +CREATE TABLE quota_usage ( + id SERIAL PRIMARY KEY NOT NULL, + reference VARCHAR(255) NOT NULL, + reference_id VARCHAR(255) NOT NULL, + used JSONB NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP, + UNIQUE(reference, reference_id) +); + +INSERT INTO quota (reference, reference_id, hard, creation_time, update_time) +SELECT + 'project', + CAST(project_id AS VARCHAR), + '{"count": -1, "storage": -1}', + NOW(), + NOW() +FROM + project +WHERE + deleted = 'f'; + +INSERT INTO quota_usage (id, reference, reference_id, used, creation_time, update_time) +SELECT + id, + reference, + reference_id, +'{"count": 0, "storage": 0}', + creation_time, + update_time +FROM + quota; + create table retention_policy ( id serial PRIMARY KEY NOT NULL, @@ -56,3 +141,4 @@ create table schedule update_time timestamp default CURRENT_TIMESTAMP, PRIMARY KEY (id) ); + diff --git a/src/common/config/metadata/metadatalist.go b/src/common/config/metadata/metadatalist.go index d019081ea..5dfc17ae3 100644 --- a/src/common/config/metadata/metadatalist.go +++ b/src/common/config/metadata/metadatalist.go @@ -47,6 +47,7 @@ const ( HTTPAuthGroup = "http_auth" OIDCGroup = "oidc" DatabaseGroup = "database" + QuotaGroup = "quota" // Put all config items do not belong a existing group into basic BasicGroup = "basic" ClairGroup = "clair" @@ -147,5 +148,8 @@ var ( {Name: common.WithNotary, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_NOTARY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, // the unit of expiration is minute, 43200 minutes = 30 days {Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true}, + + {Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, + {Name: common.StoragePerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "STORAGE_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, } ) diff --git a/src/common/config/metadata/type.go b/src/common/config/metadata/type.go index 6ed790c97..745f30868 100644 --- a/src/common/config/metadata/type.go +++ b/src/common/config/metadata/type.go @@ -18,9 +18,10 @@ package metadata import ( "encoding/json" "fmt" - "github.com/goharbor/harbor/src/common" "strconv" "strings" + + "github.com/goharbor/harbor/src/common" ) // Type - Use this interface to define and encapsulate the behavior of validation and transformation @@ -186,3 +187,21 @@ func (t *MapType) get(str string) (interface{}, error) { err := json.Unmarshal([]byte(str), &result) return result, err } + +// QuotaType ... +type QuotaType struct { + Int64Type +} + +func (t *QuotaType) validate(str string) error { + val, err := strconv.ParseInt(str, 10, 64) + if err != nil { + return err + } + + if val <= 0 && val != -1 { + return fmt.Errorf("quota value should be -1 or great than zero") + } + + return nil +} diff --git a/src/common/const.go b/src/common/const.go index 3826d1d2d..199de9096 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -142,4 +142,8 @@ const ( OIDCLoginPath = "/c/oidc/login" ChartUploadCtxKey = contextKey("chart_upload_event") + + // Quota setting items for project + CountPerProject = "count_per_project" + StoragePerProject = "storage_per_project" ) diff --git a/src/common/dao/artifact.go b/src/common/dao/artifact.go new file mode 100644 index 000000000..ac2b067ee --- /dev/null +++ b/src/common/dao/artifact.go @@ -0,0 +1,98 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/models" + "strings" + "time" +) + +// AddArtifact ... +func AddArtifact(af *models.Artifact) (int64, error) { + now := time.Now() + af.CreationTime = now + id, err := GetOrmer().Insert(af) + if err != nil { + if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + return 0, ErrDupRows + } + return 0, err + } + return id, nil +} + +// UpdateArtifactDigest ... +func UpdateArtifactDigest(af *models.Artifact) error { + _, err := GetOrmer().Update(af, "digest") + return err +} + +// DeleteArtifact ... +func DeleteArtifact(id int64) error { + _, err := GetOrmer().QueryTable(&models.Artifact{}).Filter("ID", id).Delete() + return err +} + +// DeleteArtifactByDigest ... +func DeleteArtifactByDigest(digest string) error { + _, err := GetOrmer().Raw(`delete from artifact where digest = ? `, digest).Exec() + if err != nil { + return err + } + return nil +} + +// DeleteByTag ... +func DeleteByTag(projectID int, repo, tag string) error { + _, err := GetOrmer().Raw(`delete from artifact where project_id = ? and repo = ? and tag = ? `, + projectID, repo, tag).Exec() + if err != nil { + return err + } + return nil +} + +// ListArtifacts list artifacts according to the query conditions +func ListArtifacts(query *models.ArtifactQuery) ([]*models.Artifact, error) { + qs := getArtifactQuerySetter(query) + if query.Size > 0 { + qs = qs.Limit(query.Size) + if query.Page > 0 { + qs = qs.Offset((query.Page - 1) * query.Size) + } + } + afs := []*models.Artifact{} + _, err := qs.All(&afs) + return afs, err +} + +func getArtifactQuerySetter(query *models.ArtifactQuery) orm.QuerySeter { + qs := GetOrmer().QueryTable(&models.Artifact{}) + if query.PID != 0 { + qs = qs.Filter("PID", query.PID) + } + if len(query.Repo) > 0 { + qs = qs.Filter("Repo", query.Repo) + } + if len(query.Tag) > 0 { + qs = qs.Filter("Tag", query.Tag) + } + if len(query.Digest) > 0 { + qs = qs.Filter("Digest", query.Digest) + } + return qs +} diff --git a/src/common/dao/artifact_blob.go b/src/common/dao/artifact_blob.go new file mode 100644 index 000000000..f1bcabb56 --- /dev/null +++ b/src/common/dao/artifact_blob.go @@ -0,0 +1,110 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "fmt" + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/pkg/errors" + "strconv" + "strings" + "time" +) + +// AddArtifactNBlob ... +func AddArtifactNBlob(afnb *models.ArtifactAndBlob) (int64, error) { + now := time.Now() + afnb.CreationTime = now + id, err := GetOrmer().Insert(afnb) + if err != nil { + if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + return 0, ErrDupRows + } + return 0, err + } + return id, nil +} + +// AddArtifactNBlobs ... +func AddArtifactNBlobs(afnbs []*models.ArtifactAndBlob) error { + o := orm.NewOrm() + err := o.Begin() + if err != nil { + return err + } + + var errInsertMultiple error + total := len(afnbs) + successNums, err := o.InsertMulti(total, afnbs) + if err != nil { + errInsertMultiple = err + if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + errInsertMultiple = errors.Wrap(errInsertMultiple, ErrDupRows.Error()) + } + err := o.Rollback() + if err != nil { + log.Errorf("fail to rollback when to insert multiple artifact and blobs, %v", err) + errInsertMultiple = errors.Wrap(errInsertMultiple, err.Error()) + } + return errInsertMultiple + } + + // part of them cannot be inserted successfully. + if successNums != int64(total) { + errInsertMultiple = errors.New("Not all of artifact and blobs are inserted successfully") + err := o.Rollback() + if err != nil { + log.Errorf("fail to rollback when to insert multiple artifact and blobs, %v", err) + errInsertMultiple = errors.Wrap(errInsertMultiple, err.Error()) + } + return errInsertMultiple + } + + err = o.Commit() + if err != nil { + log.Errorf("fail to commit when to insert multiple artifact and blobs, %v", err) + return fmt.Errorf("fail to commit when to insert multiple artifact and blobs, %v", err) + } + + return nil +} + +// DeleteArtifactAndBlobByDigest ... +func DeleteArtifactAndBlobByDigest(digest string) error { + _, err := GetOrmer().Raw(`delete from artifact_blob where digest_af = ? `, digest).Exec() + if err != nil { + return err + } + return nil +} + +// CountSizeOfArtifact ... +func CountSizeOfArtifact(digest string) (int64, error) { + var res []orm.Params + num, err := GetOrmer().Raw(`SELECT sum(bb.size) FROM artifact_blob afnb LEFT JOIN blob bb ON afnb.digest_blob = bb.digest WHERE afnb.digest_af = ? `, digest).Values(&res) + if err != nil { + return -1, err + } + if num > 0 { + size, err := strconv.ParseInt(res[0]["sum"].(string), 0, 64) + if err != nil { + return -1, err + } + return size, nil + } + return -1, err +} diff --git a/src/common/dao/artifact_blob_test.go b/src/common/dao/artifact_blob_test.go new file mode 100644 index 000000000..3da44748b --- /dev/null +++ b/src/common/dao/artifact_blob_test.go @@ -0,0 +1,131 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "testing" + + "github.com/goharbor/harbor/src/common/models" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAddArtifactNBlob(t *testing.T) { + afnb := &models.ArtifactAndBlob{ + DigestAF: "vvvv", + DigestBlob: "aaaa", + } + + // add + id, err := AddArtifactNBlob(afnb) + require.Nil(t, err) + afnb.ID = id + assert.Equal(t, id, int64(1)) +} + +func TestAddArtifactNBlobs(t *testing.T) { + afnb1 := &models.ArtifactAndBlob{ + DigestAF: "zzzz", + DigestBlob: "zzza", + } + afnb2 := &models.ArtifactAndBlob{ + DigestAF: "zzzz", + DigestBlob: "zzzb", + } + afnb3 := &models.ArtifactAndBlob{ + DigestAF: "zzzz", + DigestBlob: "zzzc", + } + + var afnbs []*models.ArtifactAndBlob + afnbs = append(afnbs, afnb1) + afnbs = append(afnbs, afnb2) + afnbs = append(afnbs, afnb3) + + // add + err := AddArtifactNBlobs(afnbs) + require.Nil(t, err) +} + +func TestDeleteArtifactAndBlobByDigest(t *testing.T) { + afnb := &models.ArtifactAndBlob{ + DigestAF: "vvvv", + DigestBlob: "vvva", + } + + // add + _, err := AddArtifactNBlob(afnb) + require.Nil(t, err) + + // delete + err = DeleteArtifactAndBlobByDigest(afnb.DigestAF) + require.Nil(t, err) +} + +func TestCountSizeOfArtifact(t *testing.T) { + + afnb1 := &models.ArtifactAndBlob{ + DigestAF: "xxxx", + DigestBlob: "aaaa", + } + afnb2 := &models.ArtifactAndBlob{ + DigestAF: "xxxx", + DigestBlob: "aaab", + } + afnb3 := &models.ArtifactAndBlob{ + DigestAF: "xxxx", + DigestBlob: "aaac", + } + + var afnbs []*models.ArtifactAndBlob + afnbs = append(afnbs, afnb1) + afnbs = append(afnbs, afnb2) + afnbs = append(afnbs, afnb3) + + err := AddArtifactNBlobs(afnbs) + require.Nil(t, err) + + blob1 := &models.Blob{ + Digest: "aaaa", + ContentType: "v2.blob", + Size: 100, + } + + _, err = AddBlob(blob1) + require.Nil(t, err) + + blob2 := &models.Blob{ + Digest: "aaab", + ContentType: "v2.blob", + Size: 200, + } + + _, err = AddBlob(blob2) + require.Nil(t, err) + + blob3 := &models.Blob{ + Digest: "aaac", + ContentType: "v2.blob", + Size: 300, + } + + _, err = AddBlob(blob3) + require.Nil(t, err) + + imageSize, err := CountSizeOfArtifact("xxxx") + require.Nil(t, err) + require.Equal(t, imageSize, int64(600)) +} diff --git a/src/common/dao/artifact_test.go b/src/common/dao/artifact_test.go new file mode 100644 index 000000000..1869f2982 --- /dev/null +++ b/src/common/dao/artifact_test.go @@ -0,0 +1,131 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "testing" + + "github.com/goharbor/harbor/src/common/models" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAddArtifact(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "hello-world", + Tag: "latest", + Digest: "1234abcd", + Kind: "image", + } + + // add + id, err := AddArtifact(af) + require.Nil(t, err) + af.ID = id + assert.Equal(t, id, int64(1)) + +} + +func TestUpdateArtifactDigest(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "hello-world", + Tag: "v2.0", + Digest: "4321abcd", + Kind: "image", + } + + // add + _, err := AddArtifact(af) + require.Nil(t, err) + + af.Digest = "update_4321abcd" + require.Nil(t, UpdateArtifactDigest(af)) + assert.Equal(t, af.Digest, "update_4321abcd") +} + +func TestDeleteArtifact(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "hello-world", + Tag: "v1.0", + Digest: "1234abcd", + Kind: "image", + } + // add + id, err := AddArtifact(af) + require.Nil(t, err) + + // delete + err = DeleteArtifact(id) + require.Nil(t, err) +} + +func TestDeleteArtifactByDigest(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "hello-world", + Tag: "v1.1", + Digest: "TestDeleteArtifactByDigest", + Kind: "image", + } + // add + _, err := AddArtifact(af) + require.Nil(t, err) + + // delete + err = DeleteArtifactByDigest(af.Digest) + require.Nil(t, err) +} + +func TestDeleteArtifactByTag(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "hello-world", + Tag: "v1.2", + Digest: "TestDeleteArtifactByTag", + Kind: "image", + } + // add + _, err := AddArtifact(af) + require.Nil(t, err) + + // delete + err = DeleteByTag(1, "hello-world", "v1.2") + require.Nil(t, err) +} + +func TestListArtifacts(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "hello-world", + Tag: "v3.0", + Digest: "TestListArtifacts", + Kind: "image", + } + // add + _, err := AddArtifact(af) + require.Nil(t, err) + + afs, err := ListArtifacts(&models.ArtifactQuery{ + PID: 1, + Repo: "hello-world", + Tag: "v3.0", + }) + require.Nil(t, err) + assert.Equal(t, 1, len(afs)) +} diff --git a/src/common/dao/base.go b/src/common/dao/base.go index f1d3d2386..804a73208 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -188,3 +188,29 @@ func Escape(str string) string { str = strings.Replace(str, `_`, `\_`, -1) return str } + +// WithTransaction helper for transaction +func WithTransaction(handler func(o orm.Ormer) error) error { + o := orm.NewOrm() + + if err := o.Begin(); err != nil { + log.Errorf("begin transaction failed: %v", err) + return err + } + + if err := handler(o); err != nil { + if e := o.Rollback(); e != nil { + log.Errorf("rollback transaction failed: %v", e) + return e + } + + return err + } + + if err := o.Commit(); err != nil { + log.Errorf("commit transaction failed: %v", err) + return err + } + + return nil +} diff --git a/src/common/dao/blob.go b/src/common/dao/blob.go new file mode 100644 index 000000000..9a50bc3bd --- /dev/null +++ b/src/common/dao/blob.go @@ -0,0 +1,64 @@ +package dao + +import ( + "fmt" + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + "strings" + "time" +) + +// AddBlob ... +func AddBlob(blob *models.Blob) (int64, error) { + now := time.Now() + blob.CreationTime = now + id, err := GetOrmer().Insert(blob) + if err != nil { + if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + return 0, ErrDupRows + } + return 0, err + } + return id, nil +} + +// GetBlob ... +func GetBlob(digest string) (*models.Blob, error) { + o := GetOrmer() + qs := o.QueryTable(&models.Blob{}) + qs = qs.Filter("Digest", digest) + b := []*models.Blob{} + _, err := qs.All(&b) + if err != nil { + return nil, fmt.Errorf("failed to get blob for digest %s, error: %v", digest, err) + } + if len(b) == 0 { + log.Infof("No blob found for digest %s, returning empty.", digest) + return &models.Blob{}, nil + } else if len(b) > 1 { + log.Infof("Multiple blob found for digest %s", digest) + return &models.Blob{}, fmt.Errorf("Multiple blob found for digest %s", digest) + } + return b[0], nil +} + +// DeleteBlob ... +func DeleteBlob(digest string) error { + o := GetOrmer() + _, err := o.QueryTable("blob").Filter("digest", digest).Delete() + return err +} + +// HasBlobInProject ... +func HasBlobInProject(projectID int64, digest string) (bool, error) { + var res []orm.Params + num, err := GetOrmer().Raw(`SELECT * FROM artifact af LEFT JOIN artifact_blob afnb ON af.digest = afnb.digest_af WHERE af.project_id = ? and afnb.digest_blob = ? `, projectID, digest).Values(&res) + if err != nil { + return false, err + } + if num == 0 { + return false, nil + } + return true, nil +} diff --git a/src/common/dao/blob_test.go b/src/common/dao/blob_test.go new file mode 100644 index 000000000..9d5403563 --- /dev/null +++ b/src/common/dao/blob_test.go @@ -0,0 +1,105 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "github.com/goharbor/harbor/src/common/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestAddBlob(t *testing.T) { + blob := &models.Blob{ + Digest: "1234abcd", + ContentType: "v2.blob", + Size: 1523, + } + + // add + _, err := AddBlob(blob) + require.Nil(t, err) +} + +func TestGetBlob(t *testing.T) { + blob := &models.Blob{ + Digest: "12345abcde", + ContentType: "v2.blob", + Size: 453, + } + + // add + id, err := AddBlob(blob) + require.Nil(t, err) + blob.ID = id + + blob2, err := GetBlob("12345abcde") + require.Nil(t, err) + assert.Equal(t, blob.Digest, blob2.Digest) + +} + +func TestDeleteBlob(t *testing.T) { + blob := &models.Blob{ + Digest: "123456abcdef", + ContentType: "v2.blob", + Size: 4543, + } + id, err := AddBlob(blob) + require.Nil(t, err) + blob.ID = id + err = DeleteBlob(blob.Digest) + require.Nil(t, err) +} + +func TestHasBlobInProject(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "TestHasBlobInProject", + Tag: "latest", + Digest: "tttt", + Kind: "image", + } + + // add + _, err := AddArtifact(af) + require.Nil(t, err) + + afnb1 := &models.ArtifactAndBlob{ + DigestAF: "tttt", + DigestBlob: "zzza", + } + afnb2 := &models.ArtifactAndBlob{ + DigestAF: "tttt", + DigestBlob: "zzzb", + } + afnb3 := &models.ArtifactAndBlob{ + DigestAF: "tttt", + DigestBlob: "zzzc", + } + + var afnbs []*models.ArtifactAndBlob + afnbs = append(afnbs, afnb1) + afnbs = append(afnbs, afnb2) + afnbs = append(afnbs, afnb3) + + // add + err = AddArtifactNBlobs(afnbs) + require.Nil(t, err) + + has, err := HasBlobInProject(1, "zzzb") + require.Nil(t, err) + assert.True(t, has) +} diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 0610fd305..bc070245a 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -47,8 +47,8 @@ func cleanByUser(username string) { o := GetOrmer() o.Begin() - err = execUpdate(o, `delete - from project_member + err = execUpdate(o, `delete + from project_member where entity_id = ( select user_id from harbor_user @@ -59,7 +59,7 @@ func cleanByUser(username string) { log.Error(err) } - err = execUpdate(o, `delete + err = execUpdate(o, `delete from project_member where project_id = ( select project_id @@ -71,8 +71,8 @@ func cleanByUser(username string) { log.Error(err) } - err = execUpdate(o, `delete - from access_log + err = execUpdate(o, `delete + from access_log where username = ? `, username) if err != nil { @@ -80,7 +80,7 @@ func cleanByUser(username string) { log.Error(err) } - err = execUpdate(o, `delete + err = execUpdate(o, `delete from access_log where project_id = ( select project_id @@ -1032,3 +1032,53 @@ func TestIsDupRecError(t *testing.T) { assert.True(t, isDupRecErr(fmt.Errorf("pq: duplicate key value violates unique constraint \"properties_k_key\""))) assert.False(t, isDupRecErr(fmt.Errorf("other error"))) } + +func TestWithTransaction(t *testing.T) { + reference := "transaction" + + quota := models.Quota{ + Reference: reference, + ReferenceID: "1", + Hard: "{}", + } + + failed := func(o orm.Ormer) error { + o.Insert("a) + + return fmt.Errorf("failed") + } + + var quotaID int64 + success := func(o orm.Ormer) error { + id, err := o.Insert("a) + if err != nil { + return err + } + + quotaID = id + return nil + } + + assert := assert.New(t) + + if assert.Error(WithTransaction(failed)) { + var quota models.Quota + quota.Reference = reference + quota.ReferenceID = "1" + err := GetOrmer().Read("a, "reference", "reference_id") + assert.Error(err) + assert.False(quota.ID != 0) + } + + if assert.Nil(WithTransaction(success)) { + var quota models.Quota + quota.Reference = reference + quota.ReferenceID = "1" + err := GetOrmer().Read("a, "reference", "reference_id") + assert.Nil(err) + assert.True(quota.ID != 0) + assert.Equal(quotaID, quota.ID) + + GetOrmer().Delete(&models.Quota{ID: quotaID}, "id") + } +} diff --git a/src/common/dao/project/projectmember.go b/src/common/dao/project/projectmember.go index 6776143ad..081b036f0 100644 --- a/src/common/dao/project/projectmember.go +++ b/src/common/dao/project/projectmember.go @@ -30,13 +30,13 @@ func GetProjectMember(queryMember models.Member) ([]*models.Member, error) { } o := dao.GetOrmer() - sql := ` select a.* from (select pm.id as id, pm.project_id as project_id, ug.id as entity_id, ug.group_name as entity_name, ug.creation_time, ug.update_time, r.name as rolename, - r.role_id as role, pm.entity_type as entity_type from user_group ug join project_member pm + sql := ` select a.* from (select pm.id as id, pm.project_id as project_id, ug.id as entity_id, ug.group_name as entity_name, ug.creation_time, ug.update_time, r.name as rolename, + r.role_id as role, pm.entity_type as entity_type from user_group ug join project_member pm on pm.project_id = ? and ug.id = pm.entity_id join role r on pm.role = r.role_id where pm.entity_type = 'g' union - select pm.id as id, pm.project_id as project_id, u.user_id as entity_id, u.username as entity_name, u.creation_time, u.update_time, r.name as rolename, - r.role_id as role, pm.entity_type as entity_type from harbor_user u join project_member pm - on pm.project_id = ? and u.user_id = pm.entity_id + select pm.id as id, pm.project_id as project_id, u.user_id as entity_id, u.username as entity_name, u.creation_time, u.update_time, r.name as rolename, + r.role_id as role, pm.entity_type as entity_type from harbor_user u join project_member pm + on pm.project_id = ? and u.user_id = pm.entity_id join role r on pm.role = r.role_id where u.deleted = false and pm.entity_type = 'u') as a where a.project_id = ? ` queryParam := make([]interface{}, 1) @@ -70,6 +70,27 @@ func GetProjectMember(queryMember models.Member) ([]*models.Member, error) { return members, err } +// GetTotalOfProjectMembers returns total of project members +func GetTotalOfProjectMembers(projectID int64, roles ...int) (int64, error) { + log.Debugf("Query condition %+v", projectID) + if projectID == 0 { + return 0, fmt.Errorf("failed to get total of project members, project id required %v", projectID) + } + + sql := "SELECT COUNT(1) FROM project_member WHERE project_id = ?" + + queryParam := []interface{}{projectID} + + if len(roles) > 0 { + sql += " AND role = ?" + queryParam = append(queryParam, roles[0]) + } + + var count int64 + err := dao.GetOrmer().Raw(sql, queryParam).QueryRow(&count) + return count, err +} + // AddProjectMember inserts a record to table project_member func AddProjectMember(member models.Member) (int, error) { @@ -120,23 +141,23 @@ func DeleteProjectMemberByID(pmid int) error { // SearchMemberByName search members of the project by entity_name func SearchMemberByName(projectID int64, entityName string) ([]*models.Member, error) { o := dao.GetOrmer() - sql := `select pm.id, pm.project_id, - u.username as entity_name, + sql := `select pm.id, pm.project_id, + u.username as entity_name, r.name as rolename, - pm.role, pm.entity_id, pm.entity_type + pm.role, pm.entity_id, pm.entity_type from project_member pm left join harbor_user u on pm.entity_id = u.user_id and pm.entity_type = 'u' left join role r on pm.role = r.role_id - where u.deleted = false and pm.project_id = ? and u.username like ? + where u.deleted = false and pm.project_id = ? and u.username like ? union - select pm.id, pm.project_id, - ug.group_name as entity_name, + select pm.id, pm.project_id, + ug.group_name as entity_name, r.name as rolename, - pm.role, pm.entity_id, pm.entity_type + pm.role, pm.entity_id, pm.entity_type from project_member pm left join user_group ug on pm.entity_id = ug.id and pm.entity_type = 'g' left join role r on pm.role = r.role_id - where pm.project_id = ? and ug.group_name like ? + where pm.project_id = ? and ug.group_name like ? order by entity_name ` queryParam := make([]interface{}, 4) queryParam = append(queryParam, projectID) diff --git a/src/common/dao/project/projectmember_test.go b/src/common/dao/project/projectmember_test.go index f19738009..fadb598b2 100644 --- a/src/common/dao/project/projectmember_test.go +++ b/src/common/dao/project/projectmember_test.go @@ -51,11 +51,18 @@ func TestMain(m *testing.M) { "update project set owner_id = (select user_id from harbor_user where username = 'member_test_01') where name = 'member_test_01'", "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_01') , (select user_id from harbor_user where username = 'member_test_01'), 'u', 1)", "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_01') , (select id from user_group where group_name = 'test_group_01'), 'g', 1)", + + "insert into harbor_user (username, email, password, realname) values ('member_test_02', 'member_test_02@example.com', '123456', 'member_test_02')", + "insert into project (name, owner_id) values ('member_test_02', 1)", + "insert into user_group (group_name, group_type, ldap_group_dn) values ('test_group_02', 1, 'CN=harbor_users,OU=sample,OU=vmware,DC=harbor,DC=com')", + "update project set owner_id = (select user_id from harbor_user where username = 'member_test_02') where name = 'member_test_02'", + "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_02') , (select user_id from harbor_user where username = 'member_test_02'), 'u', 1)", + "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_02') , (select id from user_group where group_name = 'test_group_02'), 'g', 1)", } clearSqls := []string{ - "delete from project where name='member_test_01'", - "delete from harbor_user where username='member_test_01' or username='pm_sample'", + "delete from project where name='member_test_01' or name='member_test_02'", + "delete from harbor_user where username='member_test_01' or username='member_test_02' or username='pm_sample'", "delete from user_group", "delete from project_member", } @@ -285,6 +292,39 @@ func TestGetProjectMember(t *testing.T) { } } + +func TestGetTotalOfProjectMembers(t *testing.T) { + currentProject, _ := dao.GetProjectByName("member_test_02") + + type args struct { + projectID int64 + roles []int + } + tests := []struct { + name string + args args + want int64 + wantErr bool + }{ + {"Get total of project admin", args{currentProject.ProjectID, []int{common.RoleProjectAdmin}}, 2, false}, + {"Get total of master", args{currentProject.ProjectID, []int{common.RoleMaster}}, 0, false}, + {"Get total of developer", args{currentProject.ProjectID, []int{common.RoleDeveloper}}, 0, false}, + {"Get total of guest", args{currentProject.ProjectID, []int{common.RoleGuest}}, 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetTotalOfProjectMembers(tt.args.projectID, tt.args.roles...) + if (err != nil) != tt.wantErr { + t.Errorf("GetTotalOfProjectMembers() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetTotalOfProjectMembers() = %v, want %v", got, tt.want) + } + }) + } +} + func PrepareGroupTest() { initSqls := []string{ `insert into user_group (group_name, group_type, ldap_group_dn) values ('harbor_group_01', 1, 'cn=harbor_user,dc=example,dc=com')`, diff --git a/src/common/dao/quota.go b/src/common/dao/quota.go new file mode 100644 index 000000000..6cf130d3d --- /dev/null +++ b/src/common/dao/quota.go @@ -0,0 +1,235 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota/driver" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/types" +) + +var ( + quotaOrderMap = map[string]string{ + "creation_time": "b.creation_time asc", + "+creation_time": "b.creation_time asc", + "-creation_time": "b.creation_time desc", + "update_time": "b.update_time asc", + "+update_time": "b.update_time asc", + "-update_time": "b.update_time desc", + } +) + +// AddQuota add quota to the database. +func AddQuota(quota models.Quota) (int64, error) { + now := time.Now() + quota.CreationTime = now + quota.UpdateTime = now + return GetOrmer().Insert("a) +} + +// GetQuota returns quota by id. +func GetQuota(id int64) (*models.Quota, error) { + q := models.Quota{ID: id} + err := GetOrmer().Read(&q, "ID") + if err == orm.ErrNoRows { + return nil, nil + } + return &q, err +} + +// UpdateQuota update the quota. +func UpdateQuota(quota models.Quota) error { + quota.UpdateTime = time.Now() + _, err := GetOrmer().Update("a) + return err +} + +// Quota quota mode for api +type Quota struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Ref driver.RefObject `json:"ref"` + Reference string `orm:"column(reference)" json:"-"` + ReferenceID string `orm:"column(reference_id)" json:"-"` + Hard string `orm:"column(hard);type(jsonb)" json:"-"` + Used string `orm:"column(used);type(jsonb)" json:"-"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +// MarshalJSON ... +func (q *Quota) MarshalJSON() ([]byte, error) { + hard, err := types.NewResourceList(q.Hard) + if err != nil { + return nil, err + } + + used, err := types.NewResourceList(q.Used) + if err != nil { + return nil, err + } + + type Alias Quota + return json.Marshal(&struct { + *Alias + Hard types.ResourceList `json:"hard"` + Used types.ResourceList `json:"used"` + }{ + Alias: (*Alias)(q), + Hard: hard, + Used: used, + }) +} + +// ListQuotas returns quotas by query. +func ListQuotas(query ...*models.QuotaQuery) ([]*Quota, error) { + condition, params := quotaQueryConditions(query...) + + sql := fmt.Sprintf(` +SELECT + a.id, + a.reference, + a.reference_id, + a.hard, + b.used, + b.creation_time, + b.update_time +FROM + quota AS a + JOIN quota_usage AS b ON a.id = b.id %s`, condition) + + orderBy := quotaOrderBy(query...) + if orderBy != "" { + sql += ` order by ` + orderBy + } + + if len(query) > 0 && query[0] != nil { + page, size := query[0].Page, query[0].Size + if size > 0 { + sql += ` limit ?` + params = append(params, size) + if page > 0 { + sql += ` offset ?` + params = append(params, size*(page-1)) + } + } + } + + var quotas []*Quota + if _, err := GetOrmer().Raw(sql, params).QueryRows("as); err != nil { + return nil, err + } + + for _, quota := range quotas { + d, ok := driver.Get(quota.Reference) + if !ok { + continue + } + + ref, err := d.Load(quota.ReferenceID) + if err != nil { + log.Warning(fmt.Sprintf("Load quota reference object (%s, %s) failed: %v", quota.Reference, quota.ReferenceID, err)) + continue + } + + quota.Ref = ref + } + + return quotas, nil +} + +// GetTotalOfQuotas returns total of quotas +func GetTotalOfQuotas(query ...*models.QuotaQuery) (int64, error) { + condition, params := quotaQueryConditions(query...) + sql := fmt.Sprintf("SELECT COUNT(1) FROM quota AS a JOIN quota_usage AS b ON a.id = b.id %s", condition) + + var count int64 + if err := GetOrmer().Raw(sql, params).QueryRow(&count); err != nil { + return 0, err + } + + return count, nil +} + +func quotaQueryConditions(query ...*models.QuotaQuery) (string, []interface{}) { + params := []interface{}{} + sql := "" + if len(query) == 0 || query[0] == nil { + return sql, params + } + + sql += `WHERE 1=1 ` + + q := query[0] + if q.ID != 0 { + sql += `AND a.id = ? ` + params = append(params, q.ID) + } + if q.Reference != "" { + sql += `AND a.reference = ? ` + params = append(params, q.Reference) + } + if q.ReferenceID != "" { + sql += `AND a.reference_id = ? ` + params = append(params, q.ReferenceID) + } + + if len(q.ReferenceIDs) != 0 { + sql += fmt.Sprintf(`AND a.reference_id IN (%s) `, paramPlaceholder(len(q.ReferenceIDs))) + params = append(params, q.ReferenceIDs) + } + + return sql, params +} + +func castQuantity(field string) string { + // cast -1 to max int64 when order by field + return fmt.Sprintf("CAST( (CASE WHEN (%[1]s) IS NULL THEN '0' WHEN (%[1]s) = '-1' THEN '9223372036854775807' ELSE (%[1]s) END) AS BIGINT )", field) +} + +func quotaOrderBy(query ...*models.QuotaQuery) string { + orderBy := "b.creation_time DESC" + + if len(query) > 0 && query[0] != nil && query[0].Sort != "" { + if val, ok := quotaOrderMap[query[0].Sort]; ok { + orderBy = val + } else { + sort := query[0].Sort + + order := "ASC" + if sort[0] == '-' { + order = "DESC" + sort = sort[1:] + } + + prefix := []string{"hard.", "used."} + for _, p := range prefix { + if strings.HasPrefix(sort, p) { + field := fmt.Sprintf("%s->>'%s'", strings.TrimSuffix(p, "."), strings.TrimPrefix(sort, p)) + orderBy = fmt.Sprintf("(%s) %s", castQuantity(field), order) + break + } + } + } + } + + return orderBy +} diff --git a/src/common/dao/quota_test.go b/src/common/dao/quota_test.go new file mode 100644 index 000000000..21daf10b9 --- /dev/null +++ b/src/common/dao/quota_test.go @@ -0,0 +1,143 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "testing" + "time" + + "github.com/goharbor/harbor/src/common/models" + "github.com/stretchr/testify/suite" +) + +var ( + quotaReference = "dao" + quotaUserReference = "user" + quotaHard = models.QuotaHard{"storage": 1024} + quotaHardLarger = models.QuotaHard{"storage": 2048} +) + +type QuotaDaoSuite struct { + suite.Suite +} + +func (suite *QuotaDaoSuite) equalHard(quota1 *models.Quota, quota2 *models.Quota) { + hard1, err := quota1.GetHard() + suite.Nil(err, "hard1 invalid") + + hard2, err := quota2.GetHard() + suite.Nil(err, "hard2 invalid") + + suite.Equal(hard1, hard2) +} + +func (suite *QuotaDaoSuite) TearDownTest() { + ClearTable("quota") + ClearTable("quota_usage") +} + +func (suite *QuotaDaoSuite) TestAddQuota() { + _, err1 := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}) + suite.Nil(err1) + + // Will failed for reference and reference_id should unique in db + _, err2 := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}) + suite.Error(err2) + + _, err3 := AddQuota(models.Quota{Reference: quotaUserReference, ReferenceID: "1", Hard: quotaHard.String()}) + suite.Nil(err3) +} + +func (suite *QuotaDaoSuite) TestGetQuota() { + quota1 := models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()} + id, err := AddQuota(quota1) + suite.Nil(err) + + // Get the new added quota + quota2, err := GetQuota(id) + suite.Nil(err) + suite.NotNil(quota2) + + // Get the quota which id is 10000 not found + quota3, err := GetQuota(10000) + suite.Nil(err) + suite.Nil(quota3) +} + +func (suite *QuotaDaoSuite) TestUpdateQuota() { + quota1 := models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()} + id, err := AddQuota(quota1) + suite.Nil(err) + + // Get the new added quota + quota2, err := GetQuota(id) + suite.Nil(err) + suite.equalHard("a1, quota2) + + // Update the quota + quota2.SetHard(quotaHardLarger) + time.Sleep(time.Millisecond * 10) // Ensure that UpdateTime changed + suite.Nil(UpdateQuota(*quota2)) + + // Get the updated quota + quota3, err := GetQuota(id) + suite.Nil(err) + suite.equalHard(quota2, quota3) + suite.NotEqual(quota2.UpdateTime, quota3.UpdateTime) +} + +func (suite *QuotaDaoSuite) TestListQuotas() { + id1, _ := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}) + AddQuotaUsage(models.QuotaUsage{ID: id1, Reference: quotaReference, ReferenceID: "1", Used: "{}"}) + + id2, _ := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "2", Hard: quotaHard.String()}) + AddQuotaUsage(models.QuotaUsage{ID: id2, Reference: quotaReference, ReferenceID: "2", Used: "{}"}) + + id3, _ := AddQuota(models.Quota{Reference: quotaUserReference, ReferenceID: "1", Hard: quotaHardLarger.String()}) + AddQuotaUsage(models.QuotaUsage{ID: id3, Reference: quotaUserReference, ReferenceID: "1", Used: "{}"}) + + id4, _ := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "3", Hard: quotaHard.String()}) + AddQuotaUsage(models.QuotaUsage{ID: id4, Reference: quotaReference, ReferenceID: "3", Used: "{}"}) + + // List all the quotas + quotas, err := ListQuotas() + suite.Nil(err) + suite.Equal(4, len(quotas)) + suite.Equal(quotaReference, quotas[0].Reference) + + // List quotas filter by reference + quotas, err = ListQuotas(&models.QuotaQuery{Reference: quotaReference}) + suite.Nil(err) + suite.Equal(3, len(quotas)) + + // List quotas filter by reference ids + quotas, err = ListQuotas(&models.QuotaQuery{Reference: quotaReference, ReferenceIDs: []string{"1", "2"}}) + suite.Nil(err) + suite.Equal(2, len(quotas)) + + // List quotas by pagination + quotas, err = ListQuotas(&models.QuotaQuery{Pagination: models.Pagination{Size: 2}}) + suite.Nil(err) + suite.Equal(2, len(quotas)) + + // List quotas by sorting + quotas, err = ListQuotas(&models.QuotaQuery{Sorting: models.Sorting{Sort: "-hard.storage"}}) + suite.Nil(err) + suite.Equal(quotaUserReference, quotas[0].Reference) +} + +func TestRunQuotaDaoSuite(t *testing.T) { + suite.Run(t, new(QuotaDaoSuite)) +} diff --git a/src/common/dao/quota_usage.go b/src/common/dao/quota_usage.go new file mode 100644 index 000000000..8e2f7ca48 --- /dev/null +++ b/src/common/dao/quota_usage.go @@ -0,0 +1,144 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "fmt" + "strings" + "time" + + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/models" +) + +var ( + quotaUsageOrderMap = map[string]string{ + "id": "id asc", + "+id": "id asc", + "-id": "id desc", + "creation_time": "creation_time asc", + "+creation_time": "creation_time asc", + "-creation_time": "creation_time desc", + "update_time": "update_time asc", + "+update_time": "update_time asc", + "-update_time": "update_time desc", + } +) + +// AddQuotaUsage add quota usage to the database. +func AddQuotaUsage(quotaUsage models.QuotaUsage) (int64, error) { + now := time.Now() + quotaUsage.CreationTime = now + quotaUsage.UpdateTime = now + return GetOrmer().Insert("aUsage) +} + +// GetQuotaUsage returns quota usage by id. +func GetQuotaUsage(id int64) (*models.QuotaUsage, error) { + q := models.QuotaUsage{ID: id} + err := GetOrmer().Read(&q, "ID") + if err == orm.ErrNoRows { + return nil, nil + } + return &q, err +} + +// UpdateQuotaUsage update the quota usage. +func UpdateQuotaUsage(quotaUsage models.QuotaUsage) error { + quotaUsage.UpdateTime = time.Now() + _, err := GetOrmer().Update("aUsage) + return err +} + +// ListQuotaUsages returns quota usages by query. +func ListQuotaUsages(query ...*models.QuotaUsageQuery) ([]*models.QuotaUsage, error) { + condition, params := quotaUsageQueryConditions(query...) + sql := fmt.Sprintf(`select * %s`, condition) + + orderBy := quotaUsageOrderBy(query...) + if orderBy != "" { + sql += ` order by ` + orderBy + } + + if len(query) > 0 && query[0] != nil { + page, size := query[0].Page, query[0].Size + if size > 0 { + sql += ` limit ?` + params = append(params, size) + if page > 0 { + sql += ` offset ?` + params = append(params, size*(page-1)) + } + } + } + + var quotaUsages []*models.QuotaUsage + if _, err := GetOrmer().Raw(sql, params).QueryRows("aUsages); err != nil { + return nil, err + } + + return quotaUsages, nil +} + +func quotaUsageQueryConditions(query ...*models.QuotaUsageQuery) (string, []interface{}) { + params := []interface{}{} + sql := `from quota_usage ` + if len(query) == 0 || query[0] == nil { + return sql, params + } + + sql += `where 1=1 ` + + q := query[0] + if q.Reference != "" { + sql += `and reference = ? ` + params = append(params, q.Reference) + } + if q.ReferenceID != "" { + sql += `and reference_id = ? ` + params = append(params, q.ReferenceID) + } + if len(q.ReferenceIDs) != 0 { + sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs))) + params = append(params, q.ReferenceIDs) + } + + return sql, params +} + +func quotaUsageOrderBy(query ...*models.QuotaUsageQuery) string { + orderBy := "" + + if len(query) > 0 && query[0] != nil && query[0].Sort != "" { + if val, ok := quotaUsageOrderMap[query[0].Sort]; ok { + orderBy = val + } else { + sort := query[0].Sort + + order := "asc" + if sort[0] == '-' { + order = "desc" + sort = sort[1:] + } + + prefix := "used." + if strings.HasPrefix(sort, prefix) { + orderBy = fmt.Sprintf("used->>'%s' %s", strings.TrimPrefix(sort, prefix), order) + } + } + } + + return orderBy +} diff --git a/src/common/dao/quota_usage_test.go b/src/common/dao/quota_usage_test.go new file mode 100644 index 000000000..40ff14124 --- /dev/null +++ b/src/common/dao/quota_usage_test.go @@ -0,0 +1,135 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "testing" + "time" + + "github.com/goharbor/harbor/src/common/models" + "github.com/stretchr/testify/suite" +) + +var ( + quotaUsageReference = "project" + quotaUsageUserReference = "user" + quotaUsageUsed = models.QuotaUsed{"storage": 1024} + quotaUsageUsedLarger = models.QuotaUsed{"storage": 2048} +) + +type QuotaUsageDaoSuite struct { + suite.Suite +} + +func (suite *QuotaUsageDaoSuite) equalUsed(usage1 *models.QuotaUsage, usage2 *models.QuotaUsage) { + used1, err := usage1.GetUsed() + suite.Nil(err, "used1 invalid") + + used2, err := usage2.GetUsed() + suite.Nil(err, "used2 invalid") + + suite.Equal(used1, used2) +} + +func (suite *QuotaUsageDaoSuite) TearDownTest() { + ClearTable("quota_usage") +} + +func (suite *QuotaUsageDaoSuite) TestAddQuotaUsage() { + _, err1 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}) + suite.Nil(err1) + + // Will failed for reference and reference_id should unique in db + _, err2 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}) + suite.Error(err2) + + _, err3 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageUserReference, ReferenceID: "1", Used: quotaUsageUsed.String()}) + suite.Nil(err3) +} + +func (suite *QuotaUsageDaoSuite) TestGetQuotaUsage() { + quotaUsage1 := models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()} + id, err := AddQuotaUsage(quotaUsage1) + suite.Nil(err) + + // Get the new added quotaUsage + quotaUsage2, err := GetQuotaUsage(id) + suite.Nil(err) + suite.NotNil(quotaUsage2) + + // Get the quotaUsage which id is 10000 not found + quotaUsage3, err := GetQuotaUsage(10000) + suite.Nil(err) + suite.Nil(quotaUsage3) +} + +func (suite *QuotaUsageDaoSuite) TestUpdateQuotaUsage() { + quotaUsage1 := models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()} + id, err := AddQuotaUsage(quotaUsage1) + suite.Nil(err) + + // Get the new added quotaUsage + quotaUsage2, err := GetQuotaUsage(id) + suite.Nil(err) + suite.equalUsed("aUsage1, quotaUsage2) + + // Update the quotaUsage + quotaUsage2.SetUsed(quotaUsageUsedLarger) + time.Sleep(time.Millisecond * 10) // Ensure that UpdateTime changed + suite.Nil(UpdateQuotaUsage(*quotaUsage2)) + + // Get the updated quotaUsage + quotaUsage3, err := GetQuotaUsage(id) + suite.Nil(err) + suite.equalUsed(quotaUsage2, quotaUsage3) + suite.NotEqual(quotaUsage2.UpdateTime, quotaUsage3.UpdateTime) +} + +func (suite *QuotaUsageDaoSuite) TestListQuotaUsages() { + AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}) + AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "2", Used: quotaUsageUsed.String()}) + AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "3", Used: quotaUsageUsed.String()}) + AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageUserReference, ReferenceID: "1", Used: quotaUsageUsedLarger.String()}) + + // List all the quotaUsages + quotaUsages, err := ListQuotaUsages() + suite.Nil(err) + suite.Equal(4, len(quotaUsages)) + suite.Equal(quotaUsageReference, quotaUsages[0].Reference) + + // List quotaUsages filter by reference + quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Reference: quotaUsageReference}) + suite.Nil(err) + suite.Equal(3, len(quotaUsages)) + + // List quotaUsages filter by reference ids + quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Reference: quotaUsageReference, ReferenceIDs: []string{"1", "2"}}) + suite.Nil(err) + suite.Equal(2, len(quotaUsages)) + + // List quotaUsages by pagination + quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Pagination: models.Pagination{Size: 2}}) + suite.Nil(err) + suite.Equal(2, len(quotaUsages)) + + // List quotaUsages by sorting + quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Sorting: models.Sorting{Sort: "-used.storage"}}) + suite.Nil(err) + suite.Equal(quotaUsageUserReference, quotaUsages[0].Reference) +} + +func TestRunQuotaUsageDaoSuite(t *testing.T) { + suite.Run(t, new(QuotaUsageDaoSuite)) +} diff --git a/src/common/models/artifact.go b/src/common/models/artifact.go new file mode 100644 index 000000000..fa6760702 --- /dev/null +++ b/src/common/models/artifact.go @@ -0,0 +1,32 @@ +package models + +import ( + "time" +) + +// Artifact holds the details of a artifact. +type Artifact struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + PID int64 `orm:"column(project_id)" json:"project_id"` + Repo string `orm:"column(repo)" json:"repo"` + Tag string `orm:"column(tag)" json:"tag"` + Digest string `orm:"column(digest)" json:"digest"` + Kind string `orm:"column(kind)" json:"kind"` + PushTime time.Time `orm:"column(push_time)" json:"push_time"` + PullTime time.Time `orm:"column(pull_time)" json:"pull_time"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` +} + +// TableName ... +func (af *Artifact) TableName() string { + return "artifact" +} + +// ArtifactQuery ... +type ArtifactQuery struct { + PID int64 + Repo string + Tag string + Digest string + Pagination +} diff --git a/src/common/models/artifact_blob.go b/src/common/models/artifact_blob.go new file mode 100644 index 000000000..a402306ee --- /dev/null +++ b/src/common/models/artifact_blob.go @@ -0,0 +1,18 @@ +package models + +import ( + "time" +) + +// ArtifactAndBlob holds the relationship between manifest and blob. +type ArtifactAndBlob struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + DigestAF string `orm:"column(digest_af)" json:"digest_af"` + DigestBlob string `orm:"column(digest_blob)" json:"digest_blob"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` +} + +// TableName ... +func (afb *ArtifactAndBlob) TableName() string { + return "artifact_blob" +} diff --git a/src/common/models/base.go b/src/common/models/base.go index f6d666ee8..7ecee503c 100644 --- a/src/common/models/base.go +++ b/src/common/models/base.go @@ -37,5 +37,11 @@ func init() { new(JobLog), new(Robot), new(OIDCUser), - new(CVEWhitelist)) + new(Blob), + new(Artifact), + new(ArtifactAndBlob), + new(CVEWhitelist), + new(Quota), + new(QuotaUsage), + ) } diff --git a/src/common/models/blob.go b/src/common/models/blob.go new file mode 100644 index 000000000..71a3c9b67 --- /dev/null +++ b/src/common/models/blob.go @@ -0,0 +1,19 @@ +package models + +import ( + "time" +) + +// Blob holds the details of a blob. +type Blob struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Digest string `orm:"column(digest)" json:"digest"` + ContentType string `orm:"column(content_type)" json:"content_type"` + Size int64 `orm:"column(size)" json:"size"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` +} + +// TableName ... +func (b *Blob) TableName() string { + return "blob" +} diff --git a/src/common/models/config.go b/src/common/models/config.go index 588e7dd84..b8c7a0e6b 100644 --- a/src/common/models/config.go +++ b/src/common/models/config.go @@ -84,6 +84,12 @@ type OIDCSetting struct { Scope []string `json:"scope"` } +// QuotaSetting wraps the settings for Quota +type QuotaSetting struct { + CountPerProject int64 `json:"count_per_project"` + StoragePerProject int64 `json:"storage_per_project"` +} + // ConfigEntry ... type ConfigEntry struct { ID int64 `orm:"pk;auto;column(id)" json:"-"` diff --git a/src/common/models/project.go b/src/common/models/project.go index 781f76268..e7f888ae1 100644 --- a/src/common/models/project.go +++ b/src/common/models/project.go @@ -17,6 +17,8 @@ package models import ( "strings" "time" + + "github.com/goharbor/harbor/src/pkg/types" ) // ProjectTable is the table name for project @@ -168,6 +170,9 @@ type ProjectRequest struct { Public *int `json:"public"` // deprecated, reserved for project creation in replication Metadata map[string]string `json:"metadata"` CVEWhitelist CVEWhitelist `json:"cve_whitelist"` + + CountLimit *int64 `json:"count_limit,omitempty"` + StorageLimit *int64 `json:"storage_limit,omitempty"` } // ProjectQueryResult ... @@ -180,3 +185,19 @@ type ProjectQueryResult struct { func (p *Project) TableName() string { return ProjectTable } + +// ProjectSummary ... +type ProjectSummary struct { + RepoCount int64 `json:"repo_count"` + ChartCount uint64 `json:"chart_count"` + + ProjectAdminCount int64 `json:"project_admin_count"` + MasterCount int64 `json:"master_count"` + DeveloperCount int64 `json:"developer_count"` + GuestCount int64 `json:"guest_count"` + + Quota struct { + Hard types.ResourceList `json:"hard"` + Used types.ResourceList `json:"used"` + } `json:"quota"` +} diff --git a/src/common/models/quota.go b/src/common/models/quota.go new file mode 100644 index 000000000..e7d8ade6e --- /dev/null +++ b/src/common/models/quota.go @@ -0,0 +1,85 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "encoding/json" + "time" + + "github.com/goharbor/harbor/src/pkg/types" +) + +// QuotaHard a map for the quota hard +type QuotaHard map[string]int64 + +func (h QuotaHard) String() string { + bytes, _ := json.Marshal(h) + return string(bytes) +} + +// Copy returns copied quota hard +func (h QuotaHard) Copy() QuotaHard { + hard := QuotaHard{} + for key, value := range h { + hard[key] = value + } + + return hard +} + +// Quota model for quota +type Quota struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Reference string `orm:"column(reference)" json:"reference"` // The reference type for quota, eg: project, user + ReferenceID string `orm:"column(reference_id)" json:"reference_id"` + Hard string `orm:"column(hard);type(jsonb)" json:"-"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +// TableName returns table name for orm +func (q *Quota) TableName() string { + return "quota" +} + +// GetHard returns quota hard +func (q *Quota) GetHard() (QuotaHard, error) { + var hard QuotaHard + if err := json.Unmarshal([]byte(q.Hard), &hard); err != nil { + return nil, err + } + + return hard, nil +} + +// SetHard set new quota hard +func (q *Quota) SetHard(hard QuotaHard) { + q.Hard = hard.String() +} + +// QuotaQuery query parameters for quota +type QuotaQuery struct { + ID int64 + Reference string + ReferenceID string + ReferenceIDs []string + Pagination + Sorting +} + +// QuotaUpdateRequest the request for quota update +type QuotaUpdateRequest struct { + Hard types.ResourceList `json:"hard"` +} diff --git a/src/common/models/quota_usage.go b/src/common/models/quota_usage.go new file mode 100644 index 000000000..c5c24eeb3 --- /dev/null +++ b/src/common/models/quota_usage.go @@ -0,0 +1,77 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "encoding/json" + "time" +) + +// QuotaUsed a map for the quota used +type QuotaUsed map[string]int64 + +func (u QuotaUsed) String() string { + bytes, _ := json.Marshal(u) + return string(bytes) +} + +// Copy returns copied quota used +func (u QuotaUsed) Copy() QuotaUsed { + used := QuotaUsed{} + for key, value := range u { + used[key] = value + } + + return used +} + +// QuotaUsage model for quota usage +type QuotaUsage struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Reference string `orm:"column(reference)" json:"reference"` // The reference type for quota usage, eg: project, user + ReferenceID string `orm:"column(reference_id)" json:"reference_id"` + Used string `orm:"column(used);type(jsonb)" json:"-"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +// TableName returns table name for orm +func (qu *QuotaUsage) TableName() string { + return "quota_usage" +} + +// GetUsed returns quota used +func (qu *QuotaUsage) GetUsed() (QuotaUsed, error) { + var used QuotaUsed + if err := json.Unmarshal([]byte(qu.Used), &used); err != nil { + return nil, err + } + + return used, nil +} + +// SetUsed set quota used +func (qu *QuotaUsage) SetUsed(used QuotaUsed) { + qu.Used = used.String() +} + +// QuotaUsageQuery query parameters for quota +type QuotaUsageQuery struct { + Reference string + ReferenceID string + ReferenceIDs []string + Pagination + Sorting +} diff --git a/src/common/quota/driver/driver.go b/src/common/quota/driver/driver.go new file mode 100644 index 000000000..fbd339e37 --- /dev/null +++ b/src/common/quota/driver/driver.go @@ -0,0 +1,59 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package driver + +import ( + "sync" + + "github.com/goharbor/harbor/src/pkg/types" +) + +var ( + driversMu sync.RWMutex + drivers = map[string]Driver{} +) + +// RefObject type for quota ref object +type RefObject map[string]interface{} + +// Driver the driver for quota +type Driver interface { + // HardLimits returns default resource list + HardLimits() types.ResourceList + // Load returns quota ref object by key + Load(key string) (RefObject, error) + // Validate validate the hard limits + Validate(hardLimits types.ResourceList) error +} + +// Register register quota driver +func Register(name string, driver Driver) { + driversMu.Lock() + defer driversMu.Unlock() + if driver == nil { + panic("quota: Register driver is nil") + } + + drivers[name] = driver +} + +// Get returns quota driver by name +func Get(name string) (Driver, bool) { + driversMu.Lock() + defer driversMu.Unlock() + + driver, ok := drivers[name] + return driver, ok +} diff --git a/src/common/quota/driver/mocks/driver.go b/src/common/quota/driver/mocks/driver.go new file mode 100644 index 000000000..8f8c1ac82 --- /dev/null +++ b/src/common/quota/driver/mocks/driver.go @@ -0,0 +1,65 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mocks + +import driver "github.com/goharbor/harbor/src/common/quota/driver" +import mock "github.com/stretchr/testify/mock" +import types "github.com/goharbor/harbor/src/pkg/types" + +// Driver is an autogenerated mock type for the Driver type +type Driver struct { + mock.Mock +} + +// HardLimits provides a mock function with given fields: +func (_m *Driver) HardLimits() types.ResourceList { + ret := _m.Called() + + var r0 types.ResourceList + if rf, ok := ret.Get(0).(func() types.ResourceList); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(types.ResourceList) + } + } + + return r0 +} + +// Load provides a mock function with given fields: key +func (_m *Driver) Load(key string) (driver.RefObject, error) { + ret := _m.Called(key) + + var r0 driver.RefObject + if rf, ok := ret.Get(0).(func(string) driver.RefObject); ok { + r0 = rf(key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(driver.RefObject) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Validate provides a mock function with given fields: resources +func (_m *Driver) Validate(resources types.ResourceList) error { + ret := _m.Called(resources) + + var r0 error + if rf, ok := ret.Get(0).(func(types.ResourceList) error); ok { + r0 = rf(resources) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/src/common/quota/driver/project/driver.go b/src/common/quota/driver/project/driver.go new file mode 100644 index 000000000..8fafded6c --- /dev/null +++ b/src/common/quota/driver/project/driver.go @@ -0,0 +1,143 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package project + +import ( + "context" + "fmt" + "strconv" + + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/config" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + dr "github.com/goharbor/harbor/src/common/quota/driver" + "github.com/goharbor/harbor/src/pkg/types" + "github.com/graph-gophers/dataloader" +) + +func init() { + dr.Register("project", newDriver()) +} + +func getProjectsBatchFn(ctx context.Context, keys dataloader.Keys) []*dataloader.Result { + handleError := func(err error) []*dataloader.Result { + var results []*dataloader.Result + var result dataloader.Result + result.Error = err + results = append(results, &result) + return results + } + + var projectIDs []int64 + for _, key := range keys { + id, err := strconv.ParseInt(key.String(), 10, 64) + if err != nil { + return handleError(err) + } + projectIDs = append(projectIDs, id) + } + + projects, err := dao.GetProjects(&models.ProjectQueryParam{}) + if err != nil { + return handleError(err) + } + + var projectsMap = make(map[int64]*models.Project, len(projectIDs)) + for _, project := range projects { + projectsMap[project.ProjectID] = project + } + + var results []*dataloader.Result + for _, projectID := range projectIDs { + project, ok := projectsMap[projectID] + if !ok { + return handleError(fmt.Errorf("project not found, "+"project_id: %d", projectID)) + } + + result := dataloader.Result{ + Data: project, + Error: nil, + } + results = append(results, &result) + } + + return results +} + +type driver struct { + cfg *config.CfgManager + loader *dataloader.Loader +} + +func (d *driver) HardLimits() types.ResourceList { + return types.ResourceList{ + types.ResourceCount: d.cfg.Get(common.CountPerProject).GetInt64(), + types.ResourceStorage: d.cfg.Get(common.StoragePerProject).GetInt64(), + } +} + +func (d *driver) Load(key string) (dr.RefObject, error) { + thunk := d.loader.Load(context.TODO(), dataloader.StringKey(key)) + + result, err := thunk() + if err != nil { + return nil, err + } + + project, ok := result.(*models.Project) + if !ok { + return nil, fmt.Errorf("bad result for project: %s", key) + } + + return dr.RefObject{ + "id": project.ProjectID, + "name": project.Name, + "owner_name": project.OwnerName, + }, nil +} + +func (d *driver) Validate(hardLimits types.ResourceList) error { + resources := map[types.ResourceName]bool{ + types.ResourceCount: true, + types.ResourceStorage: true, + } + + for resource, value := range hardLimits { + if !resources[resource] { + return fmt.Errorf("resource %s not support", resource) + } + + if value <= 0 && value != types.UNLIMITED { + return fmt.Errorf("invalid value for resource %s", resource) + } + } + + for resource := range resources { + if _, found := hardLimits[resource]; !found { + return fmt.Errorf("resource %s not found", resource) + } + } + + return nil +} + +func newDriver() dr.Driver { + cfg := config.NewDBCfgManager() + + loader := dataloader.NewBatchedLoader(getProjectsBatchFn) + + return &driver{cfg: cfg, loader: loader} +} diff --git a/src/common/quota/driver/project/driver_test.go b/src/common/quota/driver/project/driver_test.go new file mode 100644 index 000000000..992af0ae9 --- /dev/null +++ b/src/common/quota/driver/project/driver_test.go @@ -0,0 +1,77 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package project + +import ( + "os" + "testing" + + "github.com/goharbor/harbor/src/common/dao" + dr "github.com/goharbor/harbor/src/common/quota/driver" + "github.com/goharbor/harbor/src/pkg/types" + "github.com/stretchr/testify/suite" +) + +type DriverSuite struct { + suite.Suite +} + +func (suite *DriverSuite) TestHardLimits() { + driver := newDriver() + + suite.Equal(types.ResourceList{types.ResourceCount: -1, types.ResourceStorage: -1}, driver.HardLimits()) +} + +func (suite *DriverSuite) TestLoad() { + driver := newDriver() + + if ref, err := driver.Load("1"); suite.Nil(err) { + obj := dr.RefObject{ + "id": int64(1), + "name": "library", + "owner_name": "", + } + + suite.Equal(obj, ref) + } + + if ref, err := driver.Load("100000"); suite.Error(err) { + suite.Empty(ref) + } + + if ref, err := driver.Load("library"); suite.Error(err) { + suite.Empty(ref) + } +} + +func (suite *DriverSuite) TestValidate() { + driver := newDriver() + + suite.Nil(driver.Validate(types.ResourceList{types.ResourceCount: 1, types.ResourceStorage: 1024})) + suite.Error(driver.Validate(types.ResourceList{})) + suite.Error(driver.Validate(types.ResourceList{types.ResourceCount: 1})) + suite.Error(driver.Validate(types.ResourceList{types.ResourceCount: 1, types.ResourceStorage: 0})) + suite.Error(driver.Validate(types.ResourceList{types.ResourceCount: 1, types.ResourceName("foo"): 1})) +} + +func TestMain(m *testing.M) { + dao.PrepareTestForPostgresSQL() + + os.Exit(m.Run()) +} + +func TestRunDriverSuite(t *testing.T) { + suite.Run(t, new(DriverSuite)) +} diff --git a/src/common/quota/manager.go b/src/common/quota/manager.go new file mode 100644 index 000000000..43d70777b --- /dev/null +++ b/src/common/quota/manager.go @@ -0,0 +1,220 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quota + +import ( + "fmt" + "time" + + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota/driver" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/types" +) + +// Manager manager for quota +type Manager struct { + driver driver.Driver + reference string + referenceID string +} + +func (m *Manager) addQuota(o orm.Ormer, hardLimits types.ResourceList, now time.Time) (int64, error) { + quota := &models.Quota{ + Reference: m.reference, + ReferenceID: m.referenceID, + Hard: hardLimits.String(), + CreationTime: now, + UpdateTime: now, + } + + return o.Insert(quota) +} + +func (m *Manager) addUsage(o orm.Ormer, used types.ResourceList, now time.Time, ids ...int64) (int64, error) { + usage := &models.QuotaUsage{ + Reference: m.reference, + ReferenceID: m.referenceID, + Used: used.String(), + CreationTime: now, + UpdateTime: now, + } + + if len(ids) > 0 { + usage.ID = ids[0] + } + + return o.Insert(usage) +} + +func (m *Manager) newQuota(o orm.Ormer, hardLimits types.ResourceList, usages ...types.ResourceList) (int64, error) { + now := time.Now() + + id, err := m.addQuota(o, hardLimits, now) + if err != nil { + return 0, err + } + + var used types.ResourceList + if len(usages) > 0 { + used = usages[0] + } else { + used = types.Zero(hardLimits) + } + + if _, err := m.addUsage(o, used, now, id); err != nil { + return 0, err + } + + return id, nil +} + +func (m *Manager) getQuotaForUpdate(o orm.Ormer) (*models.Quota, error) { + quota := &models.Quota{Reference: m.reference, ReferenceID: m.referenceID} + if err := o.ReadForUpdate(quota, "reference", "reference_id"); err != nil { + if err == orm.ErrNoRows { + if _, err := m.newQuota(o, m.driver.HardLimits()); err != nil { + return nil, err + } + + return m.getQuotaForUpdate(o) + } + + return nil, err + } + + return quota, nil +} + +func (m *Manager) getUsageForUpdate(o orm.Ormer) (*models.QuotaUsage, error) { + usage := &models.QuotaUsage{Reference: m.reference, ReferenceID: m.referenceID} + if err := o.ReadForUpdate(usage, "reference", "reference_id"); err != nil { + return nil, err + } + + return usage, nil +} + +func (m *Manager) updateUsage(o orm.Ormer, resources types.ResourceList, + calculate func(types.ResourceList, types.ResourceList) types.ResourceList) error { + + quota, err := m.getQuotaForUpdate(o) + if err != nil { + return err + } + hardLimits, err := types.NewResourceList(quota.Hard) + if err != nil { + return err + } + + usage, err := m.getUsageForUpdate(o) + if err != nil { + return err + } + used, err := types.NewResourceList(usage.Used) + if err != nil { + return err + } + + newUsed := calculate(used, resources) + if err := isSafe(hardLimits, newUsed); err != nil { + return err + } + + usage.Used = newUsed.String() + usage.UpdateTime = time.Now() + + _, err = o.Update(usage) + return err +} + +// NewQuota create new quota for (reference, reference id) +func (m *Manager) NewQuota(hardLimit types.ResourceList, usages ...types.ResourceList) (int64, error) { + var id int64 + err := dao.WithTransaction(func(o orm.Ormer) (err error) { + id, err = m.newQuota(o, hardLimit, usages...) + return err + }) + + if err != nil { + return 0, err + } + + return id, nil +} + +// DeleteQuota delete the quota +func (m *Manager) DeleteQuota() error { + return dao.WithTransaction(func(o orm.Ormer) error { + quota := &models.Quota{Reference: m.reference, ReferenceID: m.referenceID} + if _, err := o.Delete(quota, "reference", "reference_id"); err != nil { + return err + } + + usage := &models.QuotaUsage{Reference: m.reference, ReferenceID: m.referenceID} + if _, err := o.Delete(usage, "reference", "reference_id"); err != nil { + return err + } + + return nil + }) +} + +// UpdateQuota update the quota resource spec +func (m *Manager) UpdateQuota(hardLimits types.ResourceList) error { + if err := m.driver.Validate(hardLimits); err != nil { + return err + } + + sql := `UPDATE quota SET hard = ? WHERE reference = ? AND reference_id = ?` + _, err := dao.GetOrmer().Raw(sql, hardLimits.String(), m.reference, m.referenceID).Exec() + + return err +} + +// AddResources add resources to usage +func (m *Manager) AddResources(resources types.ResourceList) error { + return dao.WithTransaction(func(o orm.Ormer) error { + return m.updateUsage(o, resources, types.Add) + }) +} + +// SubtractResources subtract resources from usage +func (m *Manager) SubtractResources(resources types.ResourceList) error { + return dao.WithTransaction(func(o orm.Ormer) error { + return m.updateUsage(o, resources, types.Subtract) + }) +} + +// NewManager returns quota manager +func NewManager(reference string, referenceID string) (*Manager, error) { + d, ok := driver.Get(reference) + if !ok { + return nil, fmt.Errorf("quota not support for %s", reference) + } + + if _, err := d.Load(referenceID); err != nil { + log.Warning(fmt.Sprintf("Load quota reference object (%s, %s) failed: %v", reference, referenceID, err)) + return nil, err + } + + return &Manager{ + driver: d, + reference: reference, + referenceID: referenceID, + }, nil +} diff --git a/src/common/quota/manager_test.go b/src/common/quota/manager_test.go new file mode 100644 index 000000000..7de96d998 --- /dev/null +++ b/src/common/quota/manager_test.go @@ -0,0 +1,295 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quota + +import ( + "fmt" + "os" + "sync" + "testing" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/quota/driver" + "github.com/goharbor/harbor/src/common/quota/driver/mocks" + "github.com/goharbor/harbor/src/pkg/types" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +var ( + hardLimits = types.ResourceList{types.ResourceCount: -1, types.ResourceStorage: 1000} + reference = "mock" +) + +func init() { + mockDriver := &mocks.Driver{} + + mockHardLimitsFn := func() types.ResourceList { + return types.ResourceList{ + types.ResourceCount: -1, + types.ResourceStorage: -1, + } + } + + mockLoadFn := func(key string) driver.RefObject { + return driver.RefObject{"id": key} + } + + mockDriver.On("HardLimits").Return(mockHardLimitsFn) + mockDriver.On("Load", mock.AnythingOfType("string")).Return(mockLoadFn, nil) + mockDriver.On("Validate", mock.AnythingOfType("types.ResourceList")).Return(nil) + + driver.Register(reference, mockDriver) +} + +func mustResourceList(s string) types.ResourceList { + resources, _ := types.NewResourceList(s) + return resources +} + +type ManagerSuite struct { + suite.Suite +} + +func (suite *ManagerSuite) SetupTest() { + _, ok := driver.Get(reference) + if !ok { + suite.Fail("driver not found for %s", reference) + } +} + +func (suite *ManagerSuite) quotaManager(referenceIDs ...string) *Manager { + referenceID := "1" + if len(referenceIDs) > 0 { + referenceID = referenceIDs[0] + } + + mgr, _ := NewManager(reference, referenceID) + return mgr +} + +func (suite *ManagerSuite) TearDownTest() { + dao.ClearTable("quota") + dao.ClearTable("quota_usage") +} + +func (suite *ManagerSuite) TestNewQuota() { + mgr := suite.quotaManager() + + if id, err := mgr.NewQuota(hardLimits); suite.Nil(err) { + quota, _ := dao.GetQuota(id) + suite.Equal(hardLimits, mustResourceList(quota.Hard)) + } + + mgr = suite.quotaManager("2") + used := types.ResourceList{types.ResourceStorage: 100} + if id, err := mgr.NewQuota(hardLimits, used); suite.Nil(err) { + quota, _ := dao.GetQuota(id) + suite.Equal(hardLimits, mustResourceList(quota.Hard)) + + usage, _ := dao.GetQuotaUsage(id) + suite.Equal(used, mustResourceList(usage.Used)) + } +} + +func (suite *ManagerSuite) TestDeleteQuota() { + mgr := suite.quotaManager() + + id, err := mgr.NewQuota(hardLimits) + if suite.Nil(err) { + quota, _ := dao.GetQuota(id) + suite.Equal(hardLimits, mustResourceList(quota.Hard)) + } + + if err := mgr.DeleteQuota(); suite.Nil(err) { + quota, _ := dao.GetQuota(id) + suite.Nil(quota) + } +} + +func (suite *ManagerSuite) TestUpdateQuota() { + mgr := suite.quotaManager() + + id, _ := mgr.NewQuota(hardLimits) + largeHardLimits := types.ResourceList{types.ResourceCount: -1, types.ResourceStorage: 1000000} + + if err := mgr.UpdateQuota(largeHardLimits); suite.Nil(err) { + quota, _ := dao.GetQuota(id) + suite.Equal(largeHardLimits, mustResourceList(quota.Hard)) + } +} + +func (suite *ManagerSuite) TestQuotaAutoCreation() { + for i := 0; i < 10; i++ { + mgr := suite.quotaManager(fmt.Sprintf("%d", i)) + resource := types.ResourceList{types.ResourceCount: 0, types.ResourceStorage: 100} + + suite.Nil(mgr.AddResources(resource)) + } +} + +func (suite *ManagerSuite) TestAddResources() { + mgr := suite.quotaManager() + id, _ := mgr.NewQuota(hardLimits) + + resource := types.ResourceList{types.ResourceCount: 0, types.ResourceStorage: 100} + + if suite.Nil(mgr.AddResources(resource)) { + usage, _ := dao.GetQuotaUsage(id) + suite.Equal(resource, mustResourceList(usage.Used)) + } + + if suite.Nil(mgr.AddResources(resource)) { + usage, _ := dao.GetQuotaUsage(id) + suite.Equal(types.ResourceList{types.ResourceCount: 0, types.ResourceStorage: 200}, mustResourceList(usage.Used)) + } + + if err := mgr.AddResources(types.ResourceList{types.ResourceStorage: 10000}); suite.Error(err) { + suite.True(IsUnsafeError(err)) + } +} + +func (suite *ManagerSuite) TestSubtractResources() { + mgr := suite.quotaManager() + id, _ := mgr.NewQuota(hardLimits) + + resource := types.ResourceList{types.ResourceCount: 0, types.ResourceStorage: 100} + + if suite.Nil(mgr.AddResources(resource)) { + usage, _ := dao.GetQuotaUsage(id) + suite.Equal(resource, mustResourceList(usage.Used)) + } + + if suite.Nil(mgr.SubtractResources(resource)) { + usage, _ := dao.GetQuotaUsage(id) + suite.Equal(types.ResourceList{types.ResourceCount: 0, types.ResourceStorage: 0}, mustResourceList(usage.Used)) + } +} + +func (suite *ManagerSuite) TestRaceAddResources() { + mgr := suite.quotaManager() + mgr.NewQuota(hardLimits) + + resources := types.ResourceList{ + types.ResourceStorage: 100, + } + + var wg sync.WaitGroup + + results := make([]bool, 100) + for i := 0; i < 100; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + results[i] = mgr.AddResources(resources) == nil + }(i) + } + wg.Wait() + + var success int + for _, result := range results { + if result { + success++ + } + } + + suite.Equal(10, success) +} + +func (suite *ManagerSuite) TestRaceSubtractResources() { + mgr := suite.quotaManager() + mgr.NewQuota(hardLimits, types.ResourceList{types.ResourceStorage: 1000}) + + resources := types.ResourceList{ + types.ResourceStorage: 100, + } + + var wg sync.WaitGroup + + results := make([]bool, 100) + for i := 0; i < 100; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + results[i] = mgr.SubtractResources(resources) == nil + }(i) + } + wg.Wait() + + var success int + for _, result := range results { + if result { + success++ + } + } + + suite.Equal(10, success) +} + +func TestMain(m *testing.M) { + dao.PrepareTestForPostgresSQL() + + if result := m.Run(); result != 0 { + os.Exit(result) + } +} + +func TestRunManagerSuite(t *testing.T) { + suite.Run(t, new(ManagerSuite)) +} + +func BenchmarkAddResources(b *testing.B) { + defer func() { + dao.ClearTable("quota") + dao.ClearTable("quota_usage") + }() + + mgr, _ := NewManager(reference, "1") + mgr.NewQuota(types.ResourceList{types.ResourceStorage: int64(b.N)}) + + resource := types.ResourceList{ + types.ResourceStorage: 1, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + mgr.AddResources(resource) + } + b.StopTimer() +} + +func BenchmarkAddResourcesParallel(b *testing.B) { + defer func() { + dao.ClearTable("quota") + dao.ClearTable("quota_usage") + }() + + mgr, _ := NewManager(reference, "1") + mgr.NewQuota(types.ResourceList{}) + + resource := types.ResourceList{ + types.ResourceStorage: 1, + } + + b.ResetTimer() + b.RunParallel(func(b *testing.PB) { + for b.Next() { + mgr.AddResources(resource) + } + }) + b.StopTimer() +} diff --git a/src/common/quota/quota.go b/src/common/quota/quota.go new file mode 100644 index 000000000..4446d61eb --- /dev/null +++ b/src/common/quota/quota.go @@ -0,0 +1,35 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quota + +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/quota/driver" + "github.com/goharbor/harbor/src/pkg/types" + + // project driver for quota + _ "github.com/goharbor/harbor/src/common/quota/driver/project" +) + +// Validate validate hard limits +func Validate(reference string, hardLimits types.ResourceList) error { + d, ok := driver.Get(reference) + if !ok { + return fmt.Errorf("quota not support for %s", reference) + } + + return d.Validate(hardLimits) +} diff --git a/src/common/quota/quota_test.go b/src/common/quota/quota_test.go new file mode 100644 index 000000000..cc089e86c --- /dev/null +++ b/src/common/quota/quota_test.go @@ -0,0 +1,45 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quota + +import ( + "testing" + + _ "github.com/goharbor/harbor/src/common/quota/driver/project" + "github.com/goharbor/harbor/src/pkg/types" +) + +func TestValidate(t *testing.T) { + type args struct { + reference string + hardLimits types.ResourceList + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"valid", args{"project", types.ResourceList{types.ResourceCount: 1, types.ResourceStorage: 1}}, false}, + {"invalid", args{"project", types.ResourceList{types.ResourceCount: 1, types.ResourceStorage: 0}}, true}, + {"not support", args{"not support", types.ResourceList{types.ResourceCount: 1}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := Validate(tt.args.reference, tt.args.hardLimits); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/src/common/quota/types.go b/src/common/quota/types.go new file mode 100644 index 000000000..35a6f60cc --- /dev/null +++ b/src/common/quota/types.go @@ -0,0 +1,32 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quota + +import ( + "github.com/goharbor/harbor/src/pkg/types" +) + +var ( + // ResourceCount alias types.ResourceCount + ResourceCount = types.ResourceCount + // ResourceStorage alias types.ResourceStorage + ResourceStorage = types.ResourceStorage +) + +// ResourceName alias types.ResourceName +type ResourceName = types.ResourceName + +// ResourceList alias types.ResourceList +type ResourceList = types.ResourceList diff --git a/src/common/quota/util.go b/src/common/quota/util.go new file mode 100644 index 000000000..33f3ce0a3 --- /dev/null +++ b/src/common/quota/util.go @@ -0,0 +1,62 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quota + +import ( + "fmt" + + "github.com/goharbor/harbor/src/pkg/types" +) + +type unsafe struct { + message string +} + +func (err *unsafe) Error() string { + return err.message +} + +func newUnsafe(message string) error { + return &unsafe{message: message} +} + +// IsUnsafeError returns true when the err is unsafe error +func IsUnsafeError(err error) bool { + _, ok := err.(*unsafe) + return ok +} + +func isSafe(hardLimits types.ResourceList, used types.ResourceList) error { + for key, value := range used { + if value < 0 { + return newUnsafe(fmt.Sprintf("bad used value: %d", value)) + } + + if hard, found := hardLimits[key]; found { + if hard == types.UNLIMITED { + continue + } + + if value > hard { + return newUnsafe(fmt.Sprintf("over the quota: used %d but only hard %d", value, hard)) + } + } else { + return newUnsafe(fmt.Sprintf("hard limit not found: %s", key)) + } + + } + + return nil +} diff --git a/src/common/quota/util_test.go b/src/common/quota/util_test.go new file mode 100644 index 000000000..806ae56af --- /dev/null +++ b/src/common/quota/util_test.go @@ -0,0 +1,96 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quota + +import ( + "errors" + "testing" + + "github.com/goharbor/harbor/src/pkg/types" +) + +func TestIsUnsafeError(t *testing.T) { + type args struct { + err error + } + tests := []struct { + name string + args args + want bool + }{ + { + "is unsafe error", + args{err: newUnsafe("unsafe")}, + true, + }, + { + "is not unsafe error", + args{err: errors.New("unsafe")}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsUnsafeError(tt.args.err); got != tt.want { + t.Errorf("IsUnsafeError() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_checkQuotas(t *testing.T) { + type args struct { + hardLimits types.ResourceList + used types.ResourceList + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + "unlimited", + args{hardLimits: types.ResourceList{types.ResourceStorage: types.UNLIMITED}, used: types.ResourceList{types.ResourceStorage: 1000}}, + false, + }, + { + "ok", + args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceStorage: 1}}, + false, + }, + { + "bad used value", + args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceStorage: -1}}, + true, + }, + { + "over the hard limit", + args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceStorage: 200}}, + true, + }, + { + "hard limit not found", + args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceCount: 1}}, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := isSafe(tt.args.hardLimits, tt.args.used); (err != nil) != tt.wantErr { + t.Errorf("isSafe() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/src/common/utils/utils.go b/src/common/utils/utils.go index cea54d342..d93c3c033 100644 --- a/src/common/utils/utils.go +++ b/src/common/utils/utils.go @@ -230,7 +230,14 @@ func GetStrValueOfAnyType(value interface{}) string { } strVal = string(b) } else { - strVal = fmt.Sprintf("%v", value) + switch val := value.(type) { + case float64: + strVal = strconv.FormatFloat(val, 'f', -1, 64) + case float32: + strVal = strconv.FormatFloat(float64(val), 'f', -1, 32) + default: + strVal = fmt.Sprintf("%v", value) + } } return strVal } diff --git a/src/common/utils/utils_test.go b/src/common/utils/utils_test.go index 66c4bca0f..206124610 100644 --- a/src/common/utils/utils_test.go +++ b/src/common/utils/utils_test.go @@ -381,3 +381,31 @@ func TestTrimLower(t *testing.T) { }) } } + +func TestGetStrValueOfAnyType(t *testing.T) { + type args struct { + value interface{} + } + tests := []struct { + name string + args args + want string + }{ + {"float", args{float32(1048576.1)}, "1048576.1"}, + {"float", args{float64(1048576.12)}, "1048576.12"}, + {"float", args{1048576.000}, "1048576"}, + {"int", args{1048576}, "1048576"}, + {"int", args{9223372036854775807}, "9223372036854775807"}, + {"string", args{"hello world"}, "hello world"}, + {"bool", args{true}, "true"}, + {"bool", args{false}, "false"}, + {"map", args{map[string]interface{}{"key1": "value1"}}, "{\"key1\":\"value1\"}"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetStrValueOfAnyType(tt.args.value); got != tt.want { + t.Errorf("GetStrValueOfAnyType() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 047641d89..2cea6bdea 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -103,6 +103,7 @@ func init() { beego.Router("/api/users/:id/permissions", &UserAPI{}, "get:ListUserPermissions") beego.Router("/api/users/:id/sysadmin", &UserAPI{}, "put:ToggleUserAdminRole") beego.Router("/api/projects/:id([0-9]+)/logs", &ProjectAPI{}, "get:Logs") + beego.Router("/api/projects/:id([0-9]+)/summary", &ProjectAPI{}, "get:Summary") beego.Router("/api/projects/:id([0-9]+)/_deletable", &ProjectAPI{}, "get:Deletable") beego.Router("/api/projects/:id([0-9]+)/metadatas/?:name", &MetadataAPI{}, "get:Get") beego.Router("/api/projects/:id([0-9]+)/metadatas/", &MetadataAPI{}, "post:Post") @@ -180,6 +181,10 @@ func init() { beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels", chartLabelAPIType, "get:GetLabels;post:MarkLabel") beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels/:id([0-9]+)", chartLabelAPIType, "delete:RemoveLabel") + quotaAPIType := &QuotaAPI{} + beego.Router("/api/quotas", quotaAPIType, "get:List") + beego.Router("/api/quotas/:id([0-9]+)", quotaAPIType, "get:Get;put:Put") + // syncRegistry if err := SyncRegistry(config.GlobalProjectMgr); err != nil { log.Fatalf("failed to sync repositories from registry: %v", err) @@ -454,6 +459,23 @@ func (a testapi) ProjectDeletable(prjUsr usrInfo, projectID int64) (int, bool, e return code, deletable.Deletable, nil } +// ProjectSummary returns summary for the project +func (a testapi) ProjectSummary(prjUsr usrInfo, projectID string) (int, apilib.ProjectSummary, error) { + _sling := sling.New().Get(a.basePath) + + // create api path + path := "api/projects/" + projectID + "/summary" + _sling = _sling.Path(path) + + var successPayload apilib.ProjectSummary + + httpStatusCode, body, err := request(_sling, jsonAcceptHeader, prjUsr) + if err == nil && httpStatusCode == 200 { + err = json.Unmarshal(body, &successPayload) + } + return httpStatusCode, successPayload, err +} + // -------------------------Member Test---------------------------------------// // Return relevant role members of projectID @@ -1213,3 +1235,55 @@ func (a testapi) RegistryUpdate(authInfo usrInfo, registryID int64, req *apimode return code, nil } + +// QuotasGet returns quotas +func (a testapi) QuotasGet(query *apilib.QuotaQuery, authInfo ...usrInfo) (int, []apilib.Quota, error) { + _sling := sling.New().Get(a.basePath). + Path("api/quotas"). + QueryStruct(query) + + var successPayload []apilib.Quota + + var httpStatusCode int + var err error + var body []byte + if len(authInfo) > 0 { + httpStatusCode, body, err = request(_sling, jsonAcceptHeader, authInfo[0]) + } else { + httpStatusCode, body, err = request(_sling, jsonAcceptHeader) + } + + if err == nil && httpStatusCode == 200 { + err = json.Unmarshal(body, &successPayload) + } else { + log.Println(string(body)) + } + + return httpStatusCode, successPayload, err +} + +// Return specific quota +func (a testapi) QuotasGetByID(authInfo usrInfo, quotaID string) (int, apilib.Quota, error) { + _sling := sling.New().Get(a.basePath) + + // create api path + path := "api/quotas/" + quotaID + _sling = _sling.Path(path) + + var successPayload apilib.Quota + + httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo) + if err == nil && httpStatusCode == 200 { + err = json.Unmarshal(body, &successPayload) + } + return httpStatusCode, successPayload, err +} + +// Update spec for the quota +func (a testapi) QuotasPut(authInfo usrInfo, quotaID string, req models.QuotaUpdateRequest) (int, error) { + path := "/api/quotas/" + quotaID + _sling := sling.New().Put(a.basePath).Path(path).BodyJSON(req) + + httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo) + return httpStatusCode, err +} diff --git a/src/core/api/project.go b/src/core/api/project.go index d6edbf23f..4a71dd316 100644 --- a/src/core/api/project.go +++ b/src/core/api/project.go @@ -19,16 +19,20 @@ import ( "net/http" "regexp" "strconv" + "sync" "time" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/dao/project" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/utils" errutil "github.com/goharbor/harbor/src/common/utils/error" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/pkg/types" "github.com/pkg/errors" ) @@ -127,6 +131,7 @@ func (p *ProjectAPI) Post() { p.SendBadRequestError(err) return } + err = validateProjectReq(pro) if err != nil { log.Errorf("Invalid project request, error: %v", err) @@ -134,6 +139,25 @@ func (p *ProjectAPI) Post() { return } + setting, err := config.QuotaSetting() + if err != nil { + log.Errorf("failed to get quota setting: %v", err) + p.SendInternalServerError(fmt.Errorf("failed to get quota setting: %v", err)) + return + } + + if !p.SecurityCtx.IsSysAdmin() { + pro.CountLimit = &setting.CountPerProject + pro.StorageLimit = &setting.StoragePerProject + } + + hardLimits, err := projectQuotaHardLimits(pro, setting) + if err != nil { + log.Errorf("Invalid project request, error: %v", err) + p.SendBadRequestError(fmt.Errorf("invalid request: %v", err)) + return + } + exist, err := p.ProjectMgr.Exists(pro.Name) if err != nil { p.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s", @@ -188,6 +212,16 @@ func (p *ProjectAPI) Post() { return } + quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10)) + if err != nil { + p.SendInternalServerError(fmt.Errorf("failed to get quota manager: %v", err)) + return + } + if _, err := quotaMgr.NewQuota(hardLimits); err != nil { + p.SendInternalServerError(fmt.Errorf("failed to create quota for project: %v", err)) + return + } + go func() { if err = dao.AddAccessLog( models.AccessLog{ @@ -233,7 +267,7 @@ func (p *ProjectAPI) Get() { err := p.populateProperties(p.project) if err != nil { - log.Errorf("populate project poroperties failed with : %+v", err) + log.Errorf("populate project properties failed with : %+v", err) } p.Data["json"] = p.project @@ -262,6 +296,16 @@ func (p *ProjectAPI) Delete() { return } + quotaMgr, err := quota.NewManager("project", strconv.FormatInt(p.project.ProjectID, 10)) + if err != nil { + p.SendInternalServerError(fmt.Errorf("failed to get quota manager: %v", err)) + return + } + if err := quotaMgr.DeleteQuota(); err != nil { + p.SendInternalServerError(fmt.Errorf("failed to delete quota for project: %v", err)) + return + } + go func() { if err := dao.AddAccessLog(models.AccessLog{ Username: p.SecurityCtx.GetUsername(), @@ -535,6 +579,37 @@ func (p *ProjectAPI) Logs() { p.ServeJSON() } +// Summary returns the summary of the project +func (p *ProjectAPI) Summary() { + if !p.requireAccess(rbac.ActionRead) { + return + } + + if err := p.populateProperties(p.project); err != nil { + log.Warningf("populate project properties failed with : %+v", err) + } + + summary := &models.ProjectSummary{ + RepoCount: p.project.RepoCount, + ChartCount: p.project.ChartCount, + } + + var wg sync.WaitGroup + for _, fn := range []func(int64, *models.ProjectSummary){getProjectQuotaSummary, getProjectMemberSummary} { + fn := fn + + wg.Add(1) + go func() { + defer wg.Done() + fn(p.project.ProjectID, summary) + }() + } + wg.Wait() + + p.Data["json"] = summary + p.ServeJSON() +} + // TODO move this to pa ckage models func validateProjectReq(req *models.ProjectRequest) error { pn := req.Name @@ -555,3 +630,71 @@ func validateProjectReq(req *models.ProjectRequest) error { req.Metadata = metas return nil } + +func projectQuotaHardLimits(req *models.ProjectRequest, setting *models.QuotaSetting) (types.ResourceList, error) { + hardLimits := types.ResourceList{} + if req.CountLimit != nil { + hardLimits[types.ResourceCount] = *req.CountLimit + } else { + hardLimits[types.ResourceCount] = setting.CountPerProject + } + + if req.StorageLimit != nil { + hardLimits[types.ResourceStorage] = *req.StorageLimit + } else { + hardLimits[types.ResourceStorage] = setting.StoragePerProject + } + + if err := quota.Validate("project", hardLimits); err != nil { + return nil, err + } + + return hardLimits, nil +} + +func getProjectQuotaSummary(projectID int64, summary *models.ProjectSummary) { + quotas, err := dao.ListQuotas(&models.QuotaQuery{Reference: "project", ReferenceID: strconv.FormatInt(projectID, 10)}) + if err != nil { + log.Debugf("failed to get quota for project: %d", projectID) + return + } + + if len(quotas) == 0 { + log.Debugf("quota not found for project: %d", projectID) + return + } + + quota := quotas[0] + + summary.Quota.Hard, _ = types.NewResourceList(quota.Hard) + summary.Quota.Used, _ = types.NewResourceList(quota.Used) +} + +func getProjectMemberSummary(projectID int64, summary *models.ProjectSummary) { + var wg sync.WaitGroup + + for _, e := range []struct { + role int + count *int64 + }{ + {common.RoleProjectAdmin, &summary.ProjectAdminCount}, + {common.RoleMaster, &summary.MasterCount}, + {common.RoleDeveloper, &summary.DeveloperCount}, + {common.RoleGuest, &summary.GuestCount}, + } { + wg.Add(1) + go func(role int, count *int64) { + defer wg.Done() + + total, err := project.GetTotalOfProjectMembers(projectID, role) + if err != nil { + log.Debugf("failed to get total of project members of role %d", role) + return + } + + *count = total + }(e.role, e.count) + } + + wg.Wait() +} diff --git a/src/core/api/project_test.go b/src/core/api/project_test.go index eb99a33d4..2ff65b2fa 100644 --- a/src/core/api/project_test.go +++ b/src/core/api/project_test.go @@ -30,6 +30,42 @@ import ( var addProject *apilib.ProjectReq var addPID int +func addProjectByName(apiTest *testapi, projectName string) (int32, error) { + req := apilib.ProjectReq{ProjectName: projectName} + code, err := apiTest.ProjectsPost(*admin, req) + if err != nil { + return 0, err + } + if code != http.StatusCreated { + return 0, fmt.Errorf("created failed") + } + + code, projects, err := apiTest.ProjectsGet(&apilib.ProjectQuery{Name: projectName}, *admin) + if err != nil { + return 0, err + } + if code != http.StatusOK { + return 0, fmt.Errorf("get failed") + } + + if len(projects) == 0 { + return 0, fmt.Errorf("oops") + } + + return projects[0].ProjectId, nil +} + +func deleteProjectByIDs(apiTest *testapi, projectIDs ...int32) error { + for _, projectID := range projectIDs { + _, err := apiTest.ProjectsDelete(*admin, fmt.Sprintf("%d", projectID)) + if err != nil { + return err + } + } + + return nil +} + func InitAddPro() { addProject = &apilib.ProjectReq{ProjectName: "add_project", Metadata: map[string]string{models.ProMetaPublic: "true"}} } @@ -90,6 +126,31 @@ func TestAddProject(t *testing.T) { assert.Equal(int(400), result, "case 4 : response code = 400 : Project name is illegal in length ") } + // case 5: response code = 201 : expect project creation with quota success. + fmt.Println("case 5 : response code = 201 : expect project creation with quota success ") + + var countLimit, storageLimit int64 + countLimit, storageLimit = 100, 10 + result, err = apiTest.ProjectsPost(*admin, apilib.ProjectReq{ProjectName: "with_quota", CountLimit: &countLimit, StorageLimit: &storageLimit}) + if err != nil { + t.Error("Error while creat project", err.Error()) + t.Log(err) + } else { + assert.Equal(int(201), result, "case 5 : response code = 201 : expect project creation with quota success ") + } + + // case 6: response code = 400 : bad quota value, create project fail + fmt.Println("case 6: response code = 400 : bad quota value, create project fail") + + countLimit, storageLimit = 100, -2 + result, err = apiTest.ProjectsPost(*admin, apilib.ProjectReq{ProjectName: "with_quota", CountLimit: &countLimit, StorageLimit: &storageLimit}) + if err != nil { + t.Error("Error while creat project", err.Error()) + t.Log(err) + } else { + assert.Equal(int(400), result, "case 6: response code = 400 : bad quota value, create project fail") + } + fmt.Printf("\n") } @@ -230,7 +291,7 @@ func TestDeleteProject(t *testing.T) { t.Error("Error while delete project", err.Error()) t.Log(err) } else { - assert.Equal(int(401), httpStatusCode, "Case 1: Project creation status should be 401") + assert.Equal(int(401), httpStatusCode, "Case 1: Project deletion status should be 401") } // --------------------------case 2: Response Code=200---------------------------------// @@ -240,7 +301,7 @@ func TestDeleteProject(t *testing.T) { t.Error("Error while delete project", err.Error()) t.Log(err) } else { - assert.Equal(int(200), httpStatusCode, "Case 2: Project creation status should be 200") + assert.Equal(int(200), httpStatusCode, "Case 2: Project deletion status should be 200") } // --------------------------case 3: Response Code=404,Project does not exist---------------------------------// @@ -251,7 +312,7 @@ func TestDeleteProject(t *testing.T) { t.Error("Error while delete project", err.Error()) t.Log(err) } else { - assert.Equal(int(404), httpStatusCode, "Case 3: Project creation status should be 404") + assert.Equal(int(404), httpStatusCode, "Case 3: Project deletion status should be 404") } // --------------------------case 4: Response Code=400,Invalid project id.---------------------------------// @@ -262,7 +323,7 @@ func TestDeleteProject(t *testing.T) { t.Error("Error while delete project", err.Error()) t.Log(err) } else { - assert.Equal(int(400), httpStatusCode, "Case 4: Project creation status should be 400") + assert.Equal(int(400), httpStatusCode, "Case 4: Project deletion status should be 400") } fmt.Printf("\n") @@ -423,3 +484,30 @@ func TestDeletable(t *testing.T) { assert.Equal(t, http.StatusOK, code) assert.False(t, del) } + +func TestProjectSummary(t *testing.T) { + fmt.Println("\nTest for Project Summary API") + assert := assert.New(t) + + apiTest := newHarborAPI() + + projectID, err := addProjectByName(apiTest, "project-summary") + assert.Nil(err) + defer func() { + deleteProjectByIDs(apiTest, projectID) + }() + + // ----------------------------case 1 : Response Code=200----------------------------// + fmt.Println("case 1: respose code:200") + httpStatusCode, summary, err := apiTest.ProjectSummary(*admin, fmt.Sprintf("%d", projectID)) + if err != nil { + t.Error("Error while search project by proName", err.Error()) + t.Log(err) + } else { + assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") + assert.Equal(int64(1), summary.ProjectAdminCount) + assert.Equal(map[string]int64{"count": -1, "storage": -1}, summary.Quota.Hard) + } + + fmt.Printf("\n") +} diff --git a/src/core/api/quota.go b/src/core/api/quota.go new file mode 100644 index 000000000..eb55a6df3 --- /dev/null +++ b/src/core/api/quota.go @@ -0,0 +1,155 @@ +// Copyright 2018 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota" + "github.com/pkg/errors" +) + +// QuotaAPI handles request to /api/quotas/ +type QuotaAPI struct { + BaseController + quota *models.Quota +} + +// Prepare validates the URL and the user +func (qa *QuotaAPI) Prepare() { + qa.BaseController.Prepare() + + if !qa.SecurityCtx.IsAuthenticated() { + qa.SendUnAuthorizedError(errors.New("Unauthorized")) + return + } + + if !qa.SecurityCtx.IsSysAdmin() { + qa.SendForbiddenError(errors.New(qa.SecurityCtx.GetUsername())) + return + } + + if len(qa.GetStringFromPath(":id")) != 0 { + id, err := qa.GetInt64FromPath(":id") + if err != nil || id <= 0 { + text := "invalid quota ID: " + if err != nil { + text += err.Error() + } else { + text += fmt.Sprintf("%d", id) + } + qa.SendBadRequestError(errors.New(text)) + return + } + + quota, err := dao.GetQuota(id) + if err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to get quota %d, error: %v", id, err)) + return + } + + if quota == nil { + qa.SendNotFoundError(fmt.Errorf("quota %d not found", id)) + return + } + + qa.quota = quota + } +} + +// Get returns quota by id +func (qa *QuotaAPI) Get() { + query := &models.QuotaQuery{ + ID: qa.quota.ID, + } + + quotas, err := dao.ListQuotas(query) + if err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to get quota %d, error: %v", qa.quota.ID, err)) + return + } + + if len(quotas) == 0 { + qa.SendNotFoundError(fmt.Errorf("quota %d not found", qa.quota.ID)) + return + } + + qa.Data["json"] = quotas[0] + qa.ServeJSON() +} + +// Put update the quota +func (qa *QuotaAPI) Put() { + var req *models.QuotaUpdateRequest + if err := qa.DecodeJSONReq(&req); err != nil { + qa.SendBadRequestError(err) + return + } + + if err := quota.Validate(qa.quota.Reference, req.Hard); err != nil { + qa.SendBadRequestError(err) + return + } + + mgr, err := quota.NewManager(qa.quota.Reference, qa.quota.ReferenceID) + if err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to create quota manager, error: %v", err)) + return + } + + if err := mgr.UpdateQuota(req.Hard); err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to update hard limits of the quota, error: %v", err)) + return + } +} + +// List returns quotas by query +func (qa *QuotaAPI) List() { + page, size, err := qa.GetPaginationParams() + if err != nil { + qa.SendBadRequestError(err) + return + } + + query := &models.QuotaQuery{ + Reference: qa.GetString("reference"), + ReferenceID: qa.GetString("reference_id"), + Pagination: models.Pagination{ + Page: page, + Size: size, + }, + Sorting: models.Sorting{ + Sort: qa.GetString("sort"), + }, + } + + total, err := dao.GetTotalOfQuotas(query) + if err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to query database for total of quotas, error: %v", err)) + return + } + + quotas, err := dao.ListQuotas(query) + if err != nil { + qa.SendInternalServerError(fmt.Errorf("failed to query database for quotas, error: %v", err)) + return + } + + qa.SetPaginationHeader(total, page, size) + qa.Data["json"] = quotas + qa.ServeJSON() +} diff --git a/src/core/api/quota_test.go b/src/core/api/quota_test.go new file mode 100644 index 000000000..ddda51457 --- /dev/null +++ b/src/core/api/quota_test.go @@ -0,0 +1,133 @@ +// Copyright 2018 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + "testing" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota" + "github.com/goharbor/harbor/src/common/quota/driver" + "github.com/goharbor/harbor/src/common/quota/driver/mocks" + "github.com/goharbor/harbor/src/pkg/types" + "github.com/goharbor/harbor/src/testing/apitests/apilib" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var ( + reference = "mock" + hardLimits = types.ResourceList{types.ResourceCount: -1, types.ResourceStorage: -1} +) + +func init() { + mockDriver := &mocks.Driver{} + + mockHardLimitsFn := func() types.ResourceList { + return hardLimits + } + + mockLoadFn := func(key string) driver.RefObject { + return driver.RefObject{"id": key} + } + + mockValidateFn := func(hardLimits types.ResourceList) error { + if len(hardLimits) == 0 { + return fmt.Errorf("no resources found") + } + + return nil + } + + mockDriver.On("HardLimits").Return(mockHardLimitsFn) + mockDriver.On("Load", mock.AnythingOfType("string")).Return(mockLoadFn, nil) + mockDriver.On("Validate", mock.AnythingOfType("types.ResourceList")).Return(mockValidateFn) + + driver.Register(reference, mockDriver) +} + +func TestQuotaAPIList(t *testing.T) { + assert := assert.New(t) + apiTest := newHarborAPI() + + count := 10 + for i := 0; i < count; i++ { + mgr, err := quota.NewManager(reference, fmt.Sprintf("%d", i)) + assert.Nil(err) + + _, err = mgr.NewQuota(hardLimits) + assert.Nil(err) + } + + code, quotas, err := apiTest.QuotasGet(&apilib.QuotaQuery{Reference: reference}, *admin) + assert.Nil(err) + assert.Equal(int(200), code) + assert.Len(quotas, count, fmt.Sprintf("quotas len should be %d", count)) + + code, quotas, err = apiTest.QuotasGet(&apilib.QuotaQuery{Reference: reference, PageSize: 1}, *admin) + assert.Nil(err) + assert.Equal(int(200), code) + assert.Len(quotas, 1) +} + +func TestQuotaAPIGet(t *testing.T) { + assert := assert.New(t) + apiTest := newHarborAPI() + + mgr, err := quota.NewManager(reference, "quota-get") + assert.Nil(err) + + quotaID, err := mgr.NewQuota(hardLimits) + assert.Nil(err) + + code, quota, err := apiTest.QuotasGetByID(*admin, fmt.Sprintf("%d", quotaID)) + assert.Nil(err) + assert.Equal(int(200), code) + assert.Equal(map[string]int64{"storage": -1, "count": -1}, quota.Hard) + + code, _, err = apiTest.QuotasGetByID(*admin, "100") + assert.Nil(err) + assert.Equal(int(404), code) +} + +func TestQuotaPut(t *testing.T) { + assert := assert.New(t) + apiTest := newHarborAPI() + + mgr, err := quota.NewManager(reference, "quota-put") + assert.Nil(err) + + quotaID, err := mgr.NewQuota(hardLimits) + assert.Nil(err) + + code, quota, err := apiTest.QuotasGetByID(*admin, fmt.Sprintf("%d", quotaID)) + assert.Nil(err) + assert.Equal(int(200), code) + assert.Equal(map[string]int64{"count": -1, "storage": -1}, quota.Hard) + + code, err = apiTest.QuotasPut(*admin, fmt.Sprintf("%d", quotaID), models.QuotaUpdateRequest{}) + assert.Nil(err, err) + assert.Equal(int(400), code) + + code, err = apiTest.QuotasPut(*admin, fmt.Sprintf("%d", quotaID), models.QuotaUpdateRequest{Hard: types.ResourceList{types.ResourceCount: 100, types.ResourceStorage: 100}}) + assert.Nil(err) + assert.Equal(int(200), code) + + code, quota, err = apiTest.QuotasGetByID(*admin, fmt.Sprintf("%d", quotaID)) + assert.Nil(err) + assert.Equal(int(200), code) + assert.Equal(map[string]int64{"count": 100, "storage": 100}, quota.Hard) +} diff --git a/src/core/config/config.go b/src/core/config/config.go index 149fc3dd2..3e04664aa 100644 --- a/src/core/config/config.go +++ b/src/core/config/config.go @@ -510,3 +510,14 @@ func OIDCSetting() (*models.OIDCSetting, error) { Scope: scope, }, nil } + +// QuotaSetting returns the setting of quota. +func QuotaSetting() (*models.QuotaSetting, error) { + if err := cfgMgr.Load(); err != nil { + return nil, err + } + return &models.QuotaSetting{ + CountPerProject: cfgMgr.Get(common.CountPerProject).GetInt64(), + StoragePerProject: cfgMgr.Get(common.StoragePerProject).GetInt64(), + }, nil +} diff --git a/src/core/controllers/controllers_test.go b/src/core/controllers/controllers_test.go index 1381a26d3..f38517ebc 100644 --- a/src/core/controllers/controllers_test.go +++ b/src/core/controllers/controllers_test.go @@ -32,7 +32,7 @@ import ( "github.com/goharbor/harbor/src/common/models" utilstest "github.com/goharbor/harbor/src/common/utils/test" "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/core/proxy" + "github.com/goharbor/harbor/src/core/middlewares" "github.com/stretchr/testify/assert" ) @@ -102,8 +102,9 @@ func TestRedirectForOIDC(t *testing.T) { // TestMain is a sample to run an endpoint test func TestAll(t *testing.T) { config.InitWithSettings(utilstest.GetUnitTestConfig()) - proxy.Init() assert := assert.New(t) + err := middlewares.Init() + assert.Nil(err) r, _ := http.NewRequest("POST", "/c/login", nil) w := httptest.NewRecorder() diff --git a/src/core/controllers/proxy.go b/src/core/controllers/proxy.go index 1ddaf9ca7..a8fe916ba 100644 --- a/src/core/controllers/proxy.go +++ b/src/core/controllers/proxy.go @@ -2,7 +2,7 @@ package controllers import ( "github.com/astaxie/beego" - "github.com/goharbor/harbor/src/core/proxy" + "github.com/goharbor/harbor/src/core/middlewares" ) // RegistryProxy is the endpoint on UI for a reverse proxy pointing to registry @@ -14,7 +14,7 @@ type RegistryProxy struct { func (p *RegistryProxy) Handle() { req := p.Ctx.Request rw := p.Ctx.ResponseWriter - proxy.Handle(rw, req) + middlewares.Handle(rw, req) } // Render ... diff --git a/src/core/main.go b/src/core/main.go index 741ee626f..3f4b4783a 100644 --- a/src/core/main.go +++ b/src/core/main.go @@ -37,7 +37,7 @@ import ( _ "github.com/goharbor/harbor/src/core/auth/uaa" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/filter" - "github.com/goharbor/harbor/src/core/proxy" + "github.com/goharbor/harbor/src/core/middlewares" "github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/pkg/scheduler" "github.com/goharbor/harbor/src/replication" @@ -166,8 +166,8 @@ func main() { } log.Info("Init proxy") - if err := proxy.Init(); err != nil { - log.Fatalf("Init proxy error: %s", err) + if err := middlewares.Init(); err != nil { + log.Fatalf("init proxy error, %v", err) } // go proxy.StartProxy() diff --git a/src/core/middlewares/chain.go b/src/core/middlewares/chain.go new file mode 100644 index 000000000..fba7a9300 --- /dev/null +++ b/src/core/middlewares/chain.go @@ -0,0 +1,72 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package middlewares + +import ( + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/contenttrust" + "github.com/goharbor/harbor/src/core/middlewares/countquota" + "github.com/goharbor/harbor/src/core/middlewares/listrepo" + "github.com/goharbor/harbor/src/core/middlewares/multiplmanifest" + "github.com/goharbor/harbor/src/core/middlewares/readonly" + "github.com/goharbor/harbor/src/core/middlewares/sizequota" + "github.com/goharbor/harbor/src/core/middlewares/url" + "github.com/goharbor/harbor/src/core/middlewares/vulnerable" + "github.com/justinas/alice" + "net/http" +) + +// DefaultCreator ... +type DefaultCreator struct { + middlewares []string +} + +// New ... +func New(middlewares []string) *DefaultCreator { + return &DefaultCreator{ + middlewares: middlewares, + } +} + +// Create creates a middleware chain ... +func (b *DefaultCreator) Create() *alice.Chain { + chain := alice.New() + for _, mName := range b.middlewares { + middlewareName := mName + chain = chain.Append(func(next http.Handler) http.Handler { + constructor := b.geMiddleware(middlewareName) + if constructor == nil { + log.Errorf("cannot init middle %s", middlewareName) + return nil + } + return constructor(next) + }) + } + return &chain +} + +func (b *DefaultCreator) geMiddleware(mName string) alice.Constructor { + middlewares := map[string]alice.Constructor{ + READONLY: func(next http.Handler) http.Handler { return readonly.New(next) }, + URL: func(next http.Handler) http.Handler { return url.New(next) }, + MUITIPLEMANIFEST: func(next http.Handler) http.Handler { return multiplmanifest.New(next) }, + LISTREPO: func(next http.Handler) http.Handler { return listrepo.New(next) }, + CONTENTTRUST: func(next http.Handler) http.Handler { return contenttrust.New(next) }, + VULNERABLE: func(next http.Handler) http.Handler { return vulnerable.New(next) }, + SIZEQUOTA: func(next http.Handler) http.Handler { return sizequota.New(next) }, + COUNTQUOTA: func(next http.Handler) http.Handler { return countquota.New(next) }, + } + return middlewares[mName] +} diff --git a/src/core/middlewares/config.go b/src/core/middlewares/config.go new file mode 100644 index 000000000..fb9ea5f75 --- /dev/null +++ b/src/core/middlewares/config.go @@ -0,0 +1,30 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package middlewares + +// const variables +const ( + READONLY = "readonly" + URL = "url" + MUITIPLEMANIFEST = "manifest" + LISTREPO = "listrepo" + CONTENTTRUST = "contenttrust" + VULNERABLE = "vulnerable" + SIZEQUOTA = "sizequota" + COUNTQUOTA = "countquota" +) + +// Middlewares with sequential organization +var Middlewares = []string{READONLY, URL, MUITIPLEMANIFEST, LISTREPO, CONTENTTRUST, VULNERABLE, SIZEQUOTA, COUNTQUOTA} diff --git a/src/core/middlewares/contenttrust/handler.go b/src/core/middlewares/contenttrust/handler.go new file mode 100644 index 000000000..82ded4e5b --- /dev/null +++ b/src/core/middlewares/contenttrust/handler.go @@ -0,0 +1,106 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package contenttrust + +import ( + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/common/utils/notary" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" + "strings" +) + +// NotaryEndpoint ... +var NotaryEndpoint = "" + +type contentTrustHandler struct { + next http.Handler +} + +// New ... +func New(next http.Handler) http.Handler { + return &contentTrustHandler{ + next: next, + } +} + +// ServeHTTP ... +func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + imgRaw := req.Context().Value(util.ImageInfoCtxKey) + if imgRaw == nil || !config.WithNotary() { + cth.next.ServeHTTP(rw, req) + return + } + img, _ := req.Context().Value(util.ImageInfoCtxKey).(util.ImageInfo) + if img.Digest == "" { + cth.next.ServeHTTP(rw, req) + return + } + if !util.GetPolicyChecker().ContentTrustEnabled(img.ProjectName) { + cth.next.ServeHTTP(rw, req) + return + } + match, err := matchNotaryDigest(img) + if err != nil { + http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "Failed in communication with Notary please check the log"), http.StatusInternalServerError) + return + } + if !match { + log.Debugf("digest mismatch, failing the response.") + http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "The image is not signed in Notary."), http.StatusPreconditionFailed) + return + } + cth.next.ServeHTTP(rw, req) +} + +func matchNotaryDigest(img util.ImageInfo) (bool, error) { + if NotaryEndpoint == "" { + NotaryEndpoint = config.InternalNotaryEndpoint() + } + targets, err := notary.GetInternalTargets(NotaryEndpoint, util.TokenUsername, img.Repository) + if err != nil { + return false, err + } + for _, t := range targets { + if isDigest(img.Reference) { + d, err := notary.DigestFromTarget(t) + if err != nil { + return false, err + } + if img.Digest == d { + return true, nil + } + } else { + if t.Tag == img.Reference { + log.Debugf("found reference: %s in notary, try to match digest.", img.Reference) + d, err := notary.DigestFromTarget(t) + if err != nil { + return false, err + } + if img.Digest == d { + return true, nil + } + } + } + } + log.Debugf("image: %#v, not found in notary", img) + return false, nil +} + +// A sha256 is a string with 64 characters. +func isDigest(ref string) bool { + return strings.HasPrefix(ref, "sha256:") && len(ref) == 71 +} diff --git a/src/core/middlewares/contenttrust/handler_test.go b/src/core/middlewares/contenttrust/handler_test.go new file mode 100644 index 000000000..05f29ff0e --- /dev/null +++ b/src/core/middlewares/contenttrust/handler_test.go @@ -0,0 +1,69 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package contenttrust + +import ( + "github.com/goharbor/harbor/src/common" + notarytest "github.com/goharbor/harbor/src/common/utils/notary/test" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/middlewares/util" + "github.com/stretchr/testify/assert" + "net/http/httptest" + "os" + "testing" +) + +var endpoint = "10.117.4.142" +var notaryServer *httptest.Server + +var admiralEndpoint = "http://127.0.0.1:8282" +var token = "" + +func TestMain(m *testing.M) { + notaryServer = notarytest.NewNotaryServer(endpoint) + defer notaryServer.Close() + NotaryEndpoint = notaryServer.URL + var defaultConfig = map[string]interface{}{ + common.ExtEndpoint: "https://" + endpoint, + common.WithNotary: true, + common.TokenExpiration: 30, + } + config.InitWithSettings(defaultConfig) + result := m.Run() + if result != 0 { + os.Exit(result) + } +} + +func TestMatchNotaryDigest(t *testing.T) { + assert := assert.New(t) + // The data from common/utils/notary/helper_test.go + img1 := util.ImageInfo{Repository: "notary-demo/busybox", Reference: "1.0", ProjectName: "notary-demo", Digest: "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"} + img2 := util.ImageInfo{Repository: "notary-demo/busybox", Reference: "2.0", ProjectName: "notary-demo", Digest: "sha256:12345678"} + + res1, err := matchNotaryDigest(img1) + assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1) + assert.True(res1) + + res2, err := matchNotaryDigest(img2) + assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2) + assert.False(res2) +} + +func TestIsDigest(t *testing.T) { + assert := assert.New(t) + assert.False(isDigest("latest")) + assert.True(isDigest("sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7")) +} diff --git a/src/core/middlewares/countquota/handler.go b/src/core/middlewares/countquota/handler.go new file mode 100644 index 000000000..c08078177 --- /dev/null +++ b/src/core/middlewares/countquota/handler.go @@ -0,0 +1,65 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package countquota + +import ( + "fmt" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" +) + +type countQuotaHandler struct { + next http.Handler +} + +// New ... +func New(next http.Handler) http.Handler { + return &countQuotaHandler{ + next: next, + } +} + +// ServeHTTP manifest ... +func (cqh *countQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + countInteceptor := getInteceptor(req) + if countInteceptor == nil { + cqh.next.ServeHTTP(rw, req) + return + } + // handler request + if err := countInteceptor.HandleRequest(req); err != nil { + log.Warningf("Error occurred when to handle request in count quota handler: %v", err) + http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in count quota handler: %v", err)), + http.StatusInternalServerError) + return + } + cqh.next.ServeHTTP(rw, req) + + // handler response + countInteceptor.HandleResponse(*rw.(*util.CustomResponseWriter), req) +} + +func getInteceptor(req *http.Request) util.RegInterceptor { + // PUT /v2//manifests/ + matchPushMF, repository, tag := util.MatchPushManifest(req) + if matchPushMF { + mfInfo := util.MfInfo{} + mfInfo.Repository = repository + mfInfo.Tag = tag + return NewPutManifestInterceptor(&mfInfo) + } + return nil +} diff --git a/src/core/middlewares/countquota/putmanifest.go b/src/core/middlewares/countquota/putmanifest.go new file mode 100644 index 000000000..1c1f3c789 --- /dev/null +++ b/src/core/middlewares/countquota/putmanifest.go @@ -0,0 +1,211 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package countquota + +import ( + "context" + "errors" + "fmt" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota" + common_util "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/common/utils/log" + common_redis "github.com/goharbor/harbor/src/common/utils/redis" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" + "strings" + "time" +) + +// PutManifestInterceptor ... +type PutManifestInterceptor struct { + mfInfo *util.MfInfo +} + +// NewPutManifestInterceptor ... +func NewPutManifestInterceptor(mfInfo *util.MfInfo) *PutManifestInterceptor { + return &PutManifestInterceptor{ + mfInfo: mfInfo, + } +} + +// HandleRequest ... +// The context has already contain mfinfo as it was put by size quota handler. +func (pmi *PutManifestInterceptor) HandleRequest(req *http.Request) error { + mfInfo := req.Context().Value(util.MFInfokKey) + mf, ok := mfInfo.(*util.MfInfo) + if !ok { + return errors.New("failed to get manifest infor from context") + } + + tagLock, err := tryLockTag(mf) + if err != nil { + return fmt.Errorf("error occurred when to lock tag %s:%s with digest %v", mf.Repository, mf.Tag, err) + } + mf.TagLock = tagLock + + imageExist, af, err := imageExist(mf) + if err != nil { + tryFreeTag(mf) + return fmt.Errorf("error occurred when to check Manifest existence %v", err) + } + mf.Exist = imageExist + if imageExist { + if af.Digest != mf.Digest { + mf.DigestChanged = true + } + } else { + quotaRes := "a.ResourceList{ + quota.ResourceCount: 1, + } + err := util.TryRequireQuota(mf.ProjectID, quotaRes) + if err != nil { + tryFreeTag(mf) + log.Errorf("Cannot get quota for the manifest %v", err) + if err == util.ErrRequireQuota { + return err + } + return fmt.Errorf("error occurred when to require quota for the manifest %v", err) + } + mf.Quota = quotaRes + } + *req = *(req.WithContext(context.WithValue(req.Context(), util.MFInfokKey, mf))) + return nil +} + +// HandleResponse ... +func (pmi *PutManifestInterceptor) HandleResponse(rw util.CustomResponseWriter, req *http.Request) { + mfInfo := req.Context().Value(util.MFInfokKey) + mf, ok := mfInfo.(*util.MfInfo) + if !ok { + log.Error("failed to convert manifest information context into MfInfo") + return + } + defer func() { + _, err := mf.TagLock.Free() + if err != nil { + log.Errorf("Error to unlock in response handler, %v", err) + } + if err := mf.TagLock.Conn.Close(); err != nil { + log.Errorf("Error to close redis connection in response handler, %v", err) + } + }() + + // 201 + if rw.Status() == http.StatusCreated { + af := &models.Artifact{ + PID: mf.ProjectID, + Repo: mf.Repository, + Tag: mf.Tag, + Digest: mf.Digest, + PushTime: time.Now(), + Kind: "Docker-Image", + } + + // insert or update + if !mf.Exist { + _, err := dao.AddArtifact(af) + if err != nil { + log.Errorf("Error to add artifact, %v", err) + return + } + } + if mf.DigestChanged { + err := dao.UpdateArtifactDigest(af) + if err != nil { + log.Errorf("Error to add artifact, %v", err) + return + } + } + + if !mf.Exist || mf.DigestChanged { + afnbs := []*models.ArtifactAndBlob{} + self := &models.ArtifactAndBlob{ + DigestAF: mf.Digest, + DigestBlob: mf.Digest, + } + afnbs = append(afnbs, self) + for _, d := range mf.Refrerence { + afnb := &models.ArtifactAndBlob{ + DigestAF: mf.Digest, + DigestBlob: d.Digest.String(), + } + afnbs = append(afnbs, afnb) + } + if err := dao.AddArtifactNBlobs(afnbs); err != nil { + if strings.Contains(err.Error(), dao.ErrDupRows.Error()) { + log.Warning("the artifact and blobs have already in the DB, it maybe an existing image with different tag") + return + } + log.Errorf("Error to add artifact and blobs in proxy response handler, %v", err) + return + } + } + + } else if rw.Status() >= 300 || rw.Status() <= 511 { + if !mf.Exist { + success := util.TryFreeQuota(mf.ProjectID, mf.Quota) + if !success { + log.Error("error to release resource booked for the manifest") + return + } + } + } + + return +} + +// tryLockTag locks tag with redis ... +func tryLockTag(mfInfo *util.MfInfo) (*common_redis.Mutex, error) { + con, err := util.GetRegRedisCon() + if err != nil { + return nil, err + } + tagLock := common_redis.New(con, "Quota::manifest-lock::"+mfInfo.Repository+":"+mfInfo.Tag, common_util.GenerateRandomString()) + success, err := tagLock.Require() + if err != nil { + return nil, err + } + if !success { + return nil, fmt.Errorf("unable to lock tag: %s ", mfInfo.Repository+":"+mfInfo.Tag) + } + return tagLock, nil +} + +func tryFreeTag(mfInfo *util.MfInfo) { + _, err := mfInfo.TagLock.Free() + if err != nil { + log.Warningf("Error to unlock tag: %s, with error: %v ", mfInfo.Tag, err) + } +} + +// check the existence of a artifact, if exist, the method will return the artifact model +func imageExist(mfInfo *util.MfInfo) (exist bool, af *models.Artifact, err error) { + artifactQuery := &models.ArtifactQuery{ + PID: mfInfo.ProjectID, + Repo: mfInfo.Repository, + Tag: mfInfo.Tag, + } + afs, err := dao.ListArtifacts(artifactQuery) + if err != nil { + log.Errorf("Error occurred when to get project ID %v", err) + return false, nil, err + } + if len(afs) > 0 { + return true, afs[0], nil + } + return false, nil, nil +} diff --git a/src/core/middlewares/inlet.go b/src/core/middlewares/inlet.go new file mode 100644 index 000000000..65854e87a --- /dev/null +++ b/src/core/middlewares/inlet.go @@ -0,0 +1,41 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package middlewares + +import ( + "errors" + "github.com/goharbor/harbor/src/core/middlewares/registryproxy" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" +) + +var head http.Handler + +// Init initialize the Proxy instance and handler chain. +func Init() error { + ph := registryproxy.New() + if ph == nil { + return errors.New("get nil when to create proxy") + } + handlerChain := New(Middlewares).Create() + head = handlerChain.Then(ph) + return nil +} + +// Handle handles the request. +func Handle(rw http.ResponseWriter, req *http.Request) { + customResW := util.NewCustomResponseWriter(rw) + head.ServeHTTP(customResW, req) +} diff --git a/src/core/middlewares/interface.go b/src/core/middlewares/interface.go new file mode 100644 index 000000000..4ca772f43 --- /dev/null +++ b/src/core/middlewares/interface.go @@ -0,0 +1,22 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package middlewares + +import "github.com/justinas/alice" + +// ChainCreator ... +type ChainCreator interface { + Create(middlewares []string) *alice.Chain +} diff --git a/src/core/middlewares/listrepo/handler.go b/src/core/middlewares/listrepo/handler.go new file mode 100644 index 000000000..9cc2a2ae0 --- /dev/null +++ b/src/core/middlewares/listrepo/handler.go @@ -0,0 +1,104 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package listrepo + +import ( + "encoding/json" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" + "net/http/httptest" + "regexp" + "strconv" +) + +const ( + catalogURLPattern = `/v2/_catalog` +) + +type listReposHandler struct { + next http.Handler +} + +// New ... +func New(next http.Handler) http.Handler { + return &listReposHandler{ + next: next, + } +} + +// ServeHTTP ... +func (lrh listReposHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + var rec *httptest.ResponseRecorder + listReposFlag := matchListRepos(req) + if listReposFlag { + rec = httptest.NewRecorder() + lrh.next.ServeHTTP(rec, req) + if rec.Result().StatusCode != http.StatusOK { + util.CopyResp(rec, rw) + return + } + var ctlg struct { + Repositories []string `json:"repositories"` + } + decoder := json.NewDecoder(rec.Body) + if err := decoder.Decode(&ctlg); err != nil { + log.Errorf("Decode repositories error: %v", err) + util.CopyResp(rec, rw) + return + } + var entries []string + for repo := range ctlg.Repositories { + log.Debugf("the repo in the response %s", ctlg.Repositories[repo]) + exist := dao.RepositoryExists(ctlg.Repositories[repo]) + if exist { + entries = append(entries, ctlg.Repositories[repo]) + } + } + type Repos struct { + Repositories []string `json:"repositories"` + } + resp := &Repos{Repositories: entries} + respJSON, err := json.Marshal(resp) + if err != nil { + log.Errorf("Encode repositories error: %v", err) + util.CopyResp(rec, rw) + return + } + + for k, v := range rec.Header() { + rw.Header()[k] = v + } + clen := len(respJSON) + rw.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(clen)) + rw.Write(respJSON) + return + } + lrh.next.ServeHTTP(rw, req) +} + +// matchListRepos checks if the request looks like a request to list repositories. +func matchListRepos(req *http.Request) bool { + if req.Method != http.MethodGet { + return false + } + re := regexp.MustCompile(catalogURLPattern) + s := re.FindStringSubmatch(req.URL.Path) + if len(s) == 1 { + return true + } + return false +} diff --git a/src/core/middlewares/listrepo/handler_test.go b/src/core/middlewares/listrepo/handler_test.go new file mode 100644 index 000000000..70bbbeaf9 --- /dev/null +++ b/src/core/middlewares/listrepo/handler_test.go @@ -0,0 +1,37 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package listrepo + +import ( + "github.com/stretchr/testify/assert" + "net/http" + "testing" +) + +func TestMatchListRepos(t *testing.T) { + assert := assert.New(t) + req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/_catalog", nil) + res1 := matchListRepos(req1) + assert.False(res1, "%s %v is not a request to list repos", req1.Method, req1.URL) + + req2, _ := http.NewRequest("GET", "http://127.0.0.1:5000/v2/_catalog", nil) + res2 := matchListRepos(req2) + assert.True(res2, "%s %v is a request to list repos", req2.Method, req2.URL) + + req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/_catalog", nil) + res3 := matchListRepos(req3) + assert.False(res3, "%s %v is not a request to pull manifest", req3.Method, req3.URL) + +} diff --git a/src/core/middlewares/multiplmanifest/handler.go b/src/core/middlewares/multiplmanifest/handler.go new file mode 100644 index 000000000..d0126696c --- /dev/null +++ b/src/core/middlewares/multiplmanifest/handler.go @@ -0,0 +1,48 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multiplmanifest + +import ( + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" + "strings" +) + +type multipleManifestHandler struct { + next http.Handler +} + +// New ... +func New(next http.Handler) http.Handler { + return &multipleManifestHandler{ + next: next, + } +} + +// ServeHTTP The handler is responsible for blocking request to upload manifest list by docker client, which is not supported so far by Harbor. +func (mh multipleManifestHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + match, _, _ := util.MatchPushManifest(req) + if match { + contentType := req.Header.Get("Content-type") + // application/vnd.docker.distribution.manifest.list.v2+json + if strings.Contains(contentType, "manifest.list.v2") { + log.Debugf("Content-type: %s is not supported, failing the response.", contentType) + http.Error(rw, util.MarshalError("UNSUPPORTED_MEDIA_TYPE", "Manifest.list is not supported."), http.StatusUnsupportedMediaType) + return + } + } + mh.next.ServeHTTP(rw, req) +} diff --git a/src/core/middlewares/readonly/hanlder.go b/src/core/middlewares/readonly/hanlder.go new file mode 100644 index 000000000..be77ac285 --- /dev/null +++ b/src/core/middlewares/readonly/hanlder.go @@ -0,0 +1,45 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package readonly + +import ( + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" +) + +type readonlyHandler struct { + next http.Handler +} + +// New ... +func New(next http.Handler) http.Handler { + return &readonlyHandler{ + next: next, + } +} + +// ServeHTTP ... +func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + if config.ReadOnly() { + if req.Method == http.MethodDelete || req.Method == http.MethodPost || req.Method == http.MethodPatch || req.Method == http.MethodPut { + log.Warningf("The request is prohibited in readonly mode, url is: %s", req.URL.Path) + http.Error(rw, util.MarshalError("DENIED", "The system is in read only mode. Any modification is prohibited."), http.StatusForbidden) + return + } + } + rh.next.ServeHTTP(rw, req) +} diff --git a/src/core/middlewares/registryproxy/handler.go b/src/core/middlewares/registryproxy/handler.go new file mode 100644 index 000000000..72a9f02f0 --- /dev/null +++ b/src/core/middlewares/registryproxy/handler.go @@ -0,0 +1,61 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registryproxy + +import ( + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/config" + "net/http" + "net/http/httputil" + "net/url" +) + +type proxyHandler struct { + handler http.Handler +} + +// New ... +func New(urls ...string) http.Handler { + var registryURL string + var err error + if len(urls) > 1 { + log.Errorf("the parm, urls should have only 0 or 1 elements") + return nil + } + if len(urls) == 0 { + registryURL, err = config.RegistryURL() + if err != nil { + log.Error(err) + return nil + } + } else { + registryURL = urls[0] + } + targetURL, err := url.Parse(registryURL) + if err != nil { + log.Error(err) + return nil + } + + return &proxyHandler{ + handler: httputil.NewSingleHostReverseProxy(targetURL), + } + +} + +// ServeHTTP ... +func (ph proxyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + ph.handler.ServeHTTP(rw, req) +} diff --git a/src/core/middlewares/sizequota/handler.go b/src/core/middlewares/sizequota/handler.go new file mode 100644 index 000000000..e9ceddc98 --- /dev/null +++ b/src/core/middlewares/sizequota/handler.go @@ -0,0 +1,231 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizequota + +import ( + "errors" + "fmt" + "github.com/garyburd/redigo/redis" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota" + common_util "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/common/utils/log" + common_redis "github.com/goharbor/harbor/src/common/utils/redis" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" + "strings" + "time" +) + +type sizeQuotaHandler struct { + next http.Handler +} + +// New ... +func New(next http.Handler) http.Handler { + return &sizeQuotaHandler{ + next: next, + } +} + +// ServeHTTP ... +func (sqh *sizeQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + sizeInteceptor := getInteceptor(req) + if sizeInteceptor == nil { + sqh.next.ServeHTTP(rw, req) + return + } + + // handler request + if err := sizeInteceptor.HandleRequest(req); err != nil { + log.Warningf("Error occurred when to handle request in size quota handler: %v", err) + http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in size quota handler: %v", err)), + http.StatusInternalServerError) + return + } + sqh.next.ServeHTTP(rw, req) + + // handler response + sizeInteceptor.HandleResponse(*rw.(*util.CustomResponseWriter), req) +} + +func getInteceptor(req *http.Request) util.RegInterceptor { + // POST /v2//blobs/uploads/?mount=&from= + matchMountBlob, repository, mount, _ := util.MatchMountBlobURL(req) + if matchMountBlob { + bb := util.BlobInfo{} + bb.Repository = repository + bb.Digest = mount + return NewMountBlobInterceptor(&bb) + } + + // PUT /v2//blobs/uploads/?digest= + matchPutBlob, repository := util.MatchPutBlobURL(req) + if matchPutBlob { + bb := util.BlobInfo{} + bb.Repository = repository + return NewPutBlobInterceptor(&bb) + } + + // PUT /v2//manifests/ + matchPushMF, repository, tag := util.MatchPushManifest(req) + if matchPushMF { + bb := util.BlobInfo{} + mfInfo := util.MfInfo{} + bb.Repository = repository + mfInfo.Repository = repository + mfInfo.Tag = tag + return NewPutManifestInterceptor(&bb, &mfInfo) + } + + // PATCH /v2//blobs/uploads/ + matchPatchBlob, _ := util.MatchPatchBlobURL(req) + if matchPatchBlob { + return NewPatchBlobInterceptor() + } + + return nil +} + +func requireQuota(conn redis.Conn, blobInfo *util.BlobInfo) error { + projectID, err := util.GetProjectID(strings.Split(blobInfo.Repository, "/")[0]) + if err != nil { + return err + } + blobInfo.ProjectID = projectID + + digestLock, err := tryLockBlob(conn, blobInfo) + if err != nil { + log.Infof("failed to lock digest in redis, %v", err) + return err + } + blobInfo.DigestLock = digestLock + + blobExist, err := dao.HasBlobInProject(blobInfo.ProjectID, blobInfo.Digest) + if err != nil { + tryFreeBlob(blobInfo) + return err + } + blobInfo.Exist = blobExist + if blobExist { + return nil + } + + // only require quota for non existing blob. + quotaRes := "a.ResourceList{ + quota.ResourceStorage: blobInfo.Size, + } + err = util.TryRequireQuota(blobInfo.ProjectID, quotaRes) + if err != nil { + log.Infof("project id, %d, size %d", blobInfo.ProjectID, blobInfo.Size) + tryFreeBlob(blobInfo) + log.Errorf("cannot get quota for the blob %v", err) + return err + } + blobInfo.Quota = quotaRes + + return nil +} + +// HandleBlobCommon handles put blob complete request +// 1, add blob into DB if success +// 2, roll back resource if failure. +func HandleBlobCommon(rw util.CustomResponseWriter, req *http.Request) error { + bbInfo := req.Context().Value(util.BBInfokKey) + bb, ok := bbInfo.(*util.BlobInfo) + if !ok { + return errors.New("failed to convert blob information context into BBInfo") + } + defer func() { + _, err := bb.DigestLock.Free() + if err != nil { + log.Errorf("Error to unlock blob digest:%s in response handler, %v", bb.Digest, err) + } + if err := bb.DigestLock.Conn.Close(); err != nil { + log.Errorf("Error to close redis connection in put blob response handler, %v", err) + } + }() + + // Do nothing for a existing blob. + if bb.Exist { + return nil + } + + if rw.Status() == http.StatusCreated { + blob := &models.Blob{ + Digest: bb.Digest, + ContentType: bb.ContentType, + Size: bb.Size, + CreationTime: time.Now(), + } + _, err := dao.AddBlob(blob) + if err != nil { + return err + } + } else if rw.Status() >= 300 || rw.Status() <= 511 { + success := util.TryFreeQuota(bb.ProjectID, bb.Quota) + if !success { + return fmt.Errorf("Error to release resource booked for the blob, %d, digest: %s ", bb.ProjectID, bb.Digest) + } + } + return nil +} + +// tryLockBlob locks blob with redis ... +func tryLockBlob(conn redis.Conn, blobInfo *util.BlobInfo) (*common_redis.Mutex, error) { + // Quota::blob-lock::projectname::digest + digestLock := common_redis.New(conn, "Quota::blob-lock::"+strings.Split(blobInfo.Repository, "/")[0]+":"+blobInfo.Digest, common_util.GenerateRandomString()) + success, err := digestLock.Require() + if err != nil { + return nil, err + } + if !success { + return nil, fmt.Errorf("unable to lock digest: %s, %s ", blobInfo.Repository, blobInfo.Digest) + } + return digestLock, nil +} + +func tryFreeBlob(blobInfo *util.BlobInfo) { + _, err := blobInfo.DigestLock.Free() + if err != nil { + log.Warningf("Error to unlock digest: %s,%s with error: %v ", blobInfo.Repository, blobInfo.Digest, err) + } +} + +func rmBlobUploadUUID(conn redis.Conn, UUID string) (bool, error) { + exists, err := redis.Int(conn.Do("EXISTS", UUID)) + if err != nil { + return false, err + } + if exists == 1 { + res, err := redis.Int(conn.Do("DEL", UUID)) + if err != nil { + return false, err + } + return res == 1, nil + } + return true, nil +} + +// put blob path: /v2//blobs/uploads/ +func getUUID(path string) string { + if !strings.Contains(path, "/") { + log.Infof("it's not a valid path string: %s", path) + return "" + } + strs := strings.Split(path, "/") + return strs[len(strs)-1] +} diff --git a/src/core/middlewares/sizequota/handler_test.go b/src/core/middlewares/sizequota/handler_test.go new file mode 100644 index 000000000..b5231f16b --- /dev/null +++ b/src/core/middlewares/sizequota/handler_test.go @@ -0,0 +1,177 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizequota + +import ( + "context" + "fmt" + "github.com/garyburd/redigo/redis" + utilstest "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/core/middlewares/util" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" +) + +const testingRedisHost = "REDIS_HOST" + +func TestMain(m *testing.M) { + utilstest.InitDatabaseFromEnv() + rc := m.Run() + if rc != 0 { + os.Exit(rc) + } +} + +func TestGetInteceptor(t *testing.T) { + assert := assert.New(t) + req1, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + res1 := getInteceptor(req1) + + _, ok := res1.(*PutManifestInterceptor) + assert.True(ok) + + req2, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/TestGetInteceptor/14.04", nil) + res2 := getInteceptor(req2) + assert.Nil(res2) + +} + +func TestRequireQuota(t *testing.T) { + con, err := redis.Dial( + "tcp", + fmt.Sprintf("%s:%d", getRedisHost(), 6379), + redis.DialConnectTimeout(30*time.Second), + redis.DialReadTimeout(time.Minute+10*time.Second), + redis.DialWriteTimeout(10*time.Second), + ) + assert.Nil(t, err) + defer con.Close() + + assert := assert.New(t) + blobInfo := &util.BlobInfo{ + Repository: "library/test", + Digest: "sha256:abcdf123sdfefeg1246", + } + + err = requireQuota(con, blobInfo) + assert.Nil(err) + +} + +func TestGetUUID(t *testing.T) { + str1 := "test/1/2/uuid-1" + uuid1 := getUUID(str1) + assert.Equal(t, uuid1, "uuid-1") + + // not a valid path, just return empty + str2 := "test-1-2-uuid-2" + uuid2 := getUUID(str2) + assert.Equal(t, uuid2, "") +} + +func TestAddRmUUID(t *testing.T) { + con, err := redis.Dial( + "tcp", + fmt.Sprintf("%s:%d", getRedisHost(), 6379), + redis.DialConnectTimeout(30*time.Second), + redis.DialReadTimeout(time.Minute+10*time.Second), + redis.DialWriteTimeout(10*time.Second), + ) + assert.Nil(t, err) + defer con.Close() + + rmfail, err := rmBlobUploadUUID(con, "test-rm-uuid") + assert.Nil(t, err) + assert.True(t, rmfail) + + success, err := util.SetBunkSize(con, "test-rm-uuid", 1000) + assert.Nil(t, err) + assert.True(t, success) + + rmSuccess, err := rmBlobUploadUUID(con, "test-rm-uuid") + assert.Nil(t, err) + assert.True(t, rmSuccess) + +} + +func TestTryFreeLockBlob(t *testing.T) { + con, err := redis.Dial( + "tcp", + fmt.Sprintf("%s:%d", getRedisHost(), 6379), + redis.DialConnectTimeout(30*time.Second), + redis.DialReadTimeout(time.Minute+10*time.Second), + redis.DialWriteTimeout(10*time.Second), + ) + assert.Nil(t, err) + defer con.Close() + + blobInfo := util.BlobInfo{ + Repository: "lock/test", + Digest: "sha256:abcdf123sdfefeg1246", + } + + lock, err := tryLockBlob(con, &blobInfo) + assert.Nil(t, err) + blobInfo.DigestLock = lock + tryFreeBlob(&blobInfo) +} + +func TestBlobCommon(t *testing.T) { + con, err := redis.Dial( + "tcp", + fmt.Sprintf("%s:%d", getRedisHost(), 6379), + redis.DialConnectTimeout(30*time.Second), + redis.DialReadTimeout(time.Minute+10*time.Second), + redis.DialWriteTimeout(10*time.Second), + ) + assert.Nil(t, err) + defer con.Close() + + req, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + blobInfo := util.BlobInfo{ + Repository: "TestBlobCommon/test", + Digest: "sha256:abcdf12345678sdfefeg1246", + ContentType: "ContentType", + Size: 101, + Exist: false, + } + + rw := httptest.NewRecorder() + customResW := util.CustomResponseWriter{ResponseWriter: rw} + customResW.WriteHeader(201) + + lock, err := tryLockBlob(con, &blobInfo) + assert.Nil(t, err) + blobInfo.DigestLock = lock + + *req = *(req.WithContext(context.WithValue(req.Context(), util.BBInfokKey, &blobInfo))) + + err = HandleBlobCommon(customResW, req) + assert.Nil(t, err) + +} + +func getRedisHost() string { + redisHost := os.Getenv(testingRedisHost) + if redisHost == "" { + redisHost = "127.0.0.1" // for local test + } + + return redisHost +} diff --git a/src/core/middlewares/sizequota/mountblob.go b/src/core/middlewares/sizequota/mountblob.go new file mode 100644 index 000000000..8eba2ee3b --- /dev/null +++ b/src/core/middlewares/sizequota/mountblob.go @@ -0,0 +1,69 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizequota + +import ( + "context" + "fmt" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" + "strings" +) + +// MountBlobInterceptor ... +type MountBlobInterceptor struct { + blobInfo *util.BlobInfo +} + +// NewMountBlobInterceptor ... +func NewMountBlobInterceptor(blobInfo *util.BlobInfo) *MountBlobInterceptor { + return &MountBlobInterceptor{ + blobInfo: blobInfo, + } +} + +// HandleRequest ... +func (mbi *MountBlobInterceptor) HandleRequest(req *http.Request) error { + tProjectID, err := util.GetProjectID(strings.Split(mbi.blobInfo.Repository, "/")[0]) + if err != nil { + return fmt.Errorf("error occurred when to get target project: %d, %v", tProjectID, err) + } + blob, err := dao.GetBlob(mbi.blobInfo.Digest) + if err != nil { + return err + } + if blob == nil { + return fmt.Errorf("the blob in the mount request with digest: %s doesn't exist", mbi.blobInfo.Digest) + } + mbi.blobInfo.Size = blob.Size + con, err := util.GetRegRedisCon() + if err != nil { + return err + } + if err := requireQuota(con, mbi.blobInfo); err != nil { + return err + } + *req = *(req.WithContext(context.WithValue(req.Context(), util.BBInfokKey, mbi.blobInfo))) + return nil +} + +// HandleResponse ... +func (mbi *MountBlobInterceptor) HandleResponse(rw util.CustomResponseWriter, req *http.Request) { + if err := HandleBlobCommon(rw, req); err != nil { + log.Error(err) + } +} diff --git a/src/core/middlewares/sizequota/mountblob_test.go b/src/core/middlewares/sizequota/mountblob_test.go new file mode 100644 index 000000000..7d6c07cbc --- /dev/null +++ b/src/core/middlewares/sizequota/mountblob_test.go @@ -0,0 +1,85 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizequota + +import ( + "context" + "fmt" + "github.com/garyburd/redigo/redis" + "github.com/goharbor/harbor/src/core/middlewares/util" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestNewMountBlobInterceptor(t *testing.T) { + blobinfo := &util.BlobInfo{} + blobinfo.Repository = "TestNewMountBlobInterceptor/latest" + + bi := NewMountBlobInterceptor(blobinfo) + assert.NotNil(t, bi) +} + +func TestMountBlobHandleRequest(t *testing.T) { + blobInfo := util.BlobInfo{ + Repository: "TestHandleRequest/test", + Digest: "sha256:TestHandleRequest1234", + ContentType: "ContentType", + Size: 101, + Exist: false, + } + req, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + bi := NewMountBlobInterceptor(&blobInfo) + assert.NotNil(t, bi.HandleRequest(req)) +} + +func TestMountBlobHandleResponse(t *testing.T) { + con, err := redis.Dial( + "tcp", + fmt.Sprintf("%s:%d", getRedisHost(), 6379), + redis.DialConnectTimeout(30*time.Second), + redis.DialReadTimeout(time.Minute+10*time.Second), + redis.DialWriteTimeout(10*time.Second), + ) + assert.Nil(t, err) + defer con.Close() + + req, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + blobInfo := util.BlobInfo{ + Repository: "TestHandleResponse/test", + Digest: "sha256:TestHandleResponseabcdf12345678sdfefeg1246", + ContentType: "ContentType", + Size: 101, + Exist: false, + } + + rw := httptest.NewRecorder() + customResW := util.CustomResponseWriter{ResponseWriter: rw} + customResW.WriteHeader(201) + + lock, err := tryLockBlob(con, &blobInfo) + assert.Nil(t, err) + blobInfo.DigestLock = lock + + *req = *(req.WithContext(context.WithValue(req.Context(), util.BBInfokKey, &blobInfo))) + + bi := NewMountBlobInterceptor(&blobInfo) + assert.NotNil(t, bi) + + bi.HandleResponse(customResW, req) + +} diff --git a/src/core/middlewares/sizequota/patchblob.go b/src/core/middlewares/sizequota/patchblob.go new file mode 100644 index 000000000..c5ce15d63 --- /dev/null +++ b/src/core/middlewares/sizequota/patchblob.go @@ -0,0 +1,86 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizequota + +import ( + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" + "strconv" + "strings" +) + +// PatchBlobInterceptor ... +type PatchBlobInterceptor struct { +} + +// NewPatchBlobInterceptor ... +func NewPatchBlobInterceptor() *PatchBlobInterceptor { + return &PatchBlobInterceptor{} +} + +// HandleRequest do nothing for patch blob, just let the request to proxy. +func (pbi *PatchBlobInterceptor) HandleRequest(req *http.Request) error { + return nil +} + +// HandleResponse record the upload process with Range attribute, set it into redis with UUID as the key +func (pbi *PatchBlobInterceptor) HandleResponse(rw util.CustomResponseWriter, req *http.Request) { + if rw.Status() != http.StatusAccepted { + return + } + + con, err := util.GetRegRedisCon() + if err != nil { + log.Error(err) + return + } + defer con.Close() + + uuid := rw.Header().Get("Docker-Upload-UUID") + if uuid == "" { + log.Errorf("no UUID in the patch blob response, the request path %s ", req.URL.Path) + return + } + + // Range: Range indicating the current progress of the upload. + // https://github.com/opencontainers/distribution-spec/blob/master/spec.md#get-blob-upload + patchRange := rw.Header().Get("Range") + if uuid == "" { + log.Errorf("no Range in the patch blob response, the request path %s ", req.URL.Path) + return + } + + endRange := strings.Split(patchRange, "-")[1] + size, err := strconv.ParseInt(endRange, 10, 64) + // docker registry did '-1' in the response + if size > 0 { + size = size + 1 + } + if err != nil { + log.Error(err) + return + } + success, err := util.SetBunkSize(con, uuid, size) + if err != nil { + log.Error(err) + return + } + if !success { + // ToDo discuss what to do here. + log.Warningf(" T_T: Fail to set bunk: %s size: %d in redis, it causes unable to set correct quota for the artifact.", uuid, size) + } + return +} diff --git a/src/core/middlewares/sizequota/patchblob_test.go b/src/core/middlewares/sizequota/patchblob_test.go new file mode 100644 index 000000000..843b505c1 --- /dev/null +++ b/src/core/middlewares/sizequota/patchblob_test.go @@ -0,0 +1,42 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizequota + +import ( + "github.com/goharbor/harbor/src/core/middlewares/util" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "testing" +) + +func TestNewPatchBlobInterceptor(t *testing.T) { + bi := NewPatchBlobInterceptor() + assert.NotNil(t, bi) +} + +func TestPatchBlobHandleRequest(t *testing.T) { + req, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + bi := NewPatchBlobInterceptor() + assert.Nil(t, bi.HandleRequest(req)) +} + +func TestPatchBlobHandleResponse(t *testing.T) { + req, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + rw := httptest.NewRecorder() + customResW := util.CustomResponseWriter{ResponseWriter: rw} + customResW.WriteHeader(400) + NewPatchBlobInterceptor().HandleResponse(customResW, req) +} diff --git a/src/core/middlewares/sizequota/putblob.go b/src/core/middlewares/sizequota/putblob.go new file mode 100644 index 000000000..e2e75b8b3 --- /dev/null +++ b/src/core/middlewares/sizequota/putblob.go @@ -0,0 +1,83 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizequota + +import ( + "context" + "errors" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/util" + "github.com/opencontainers/go-digest" + "net/http" +) + +// PutBlobInterceptor ... +type PutBlobInterceptor struct { + blobInfo *util.BlobInfo +} + +// NewPutBlobInterceptor ... +func NewPutBlobInterceptor(blobInfo *util.BlobInfo) *PutBlobInterceptor { + return &PutBlobInterceptor{ + blobInfo: blobInfo, + } +} + +// HandleRequest ... +func (pbi *PutBlobInterceptor) HandleRequest(req *http.Request) error { + // the redis connection will be closed in the put response. + con, err := util.GetRegRedisCon() + if err != nil { + return err + } + + defer func() { + if pbi.blobInfo.UUID != "" { + _, err := rmBlobUploadUUID(con, pbi.blobInfo.UUID) + if err != nil { + log.Warningf("error occurred when remove UUID for blob, %v", err) + } + } + }() + + dgstStr := req.FormValue("digest") + if dgstStr == "" { + return errors.New("blob digest missing") + } + dgst, err := digest.Parse(dgstStr) + if err != nil { + return errors.New("blob digest parsing failed") + } + + pbi.blobInfo.Digest = dgst.String() + pbi.blobInfo.UUID = getUUID(req.URL.Path) + size, err := util.GetBlobSize(con, pbi.blobInfo.UUID) + if err != nil { + return err + } + pbi.blobInfo.Size = size + if err := requireQuota(con, pbi.blobInfo); err != nil { + return err + } + *req = *(req.WithContext(context.WithValue(req.Context(), util.BBInfokKey, pbi.blobInfo))) + return nil +} + +// HandleResponse ... +func (pbi *PutBlobInterceptor) HandleResponse(rw util.CustomResponseWriter, req *http.Request) { + if err := HandleBlobCommon(rw, req); err != nil { + log.Error(err) + } +} diff --git a/src/core/middlewares/sizequota/putblob_test.go b/src/core/middlewares/sizequota/putblob_test.go new file mode 100644 index 000000000..847623c56 --- /dev/null +++ b/src/core/middlewares/sizequota/putblob_test.go @@ -0,0 +1,80 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizequota + +import ( + "context" + "fmt" + "github.com/garyburd/redigo/redis" + "github.com/goharbor/harbor/src/core/middlewares/util" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestNewPutBlobInterceptor(t *testing.T) { + blobinfo := &util.BlobInfo{} + blobinfo.Repository = "TestNewPutBlobInterceptor/latest" + + bi := NewPutBlobInterceptor(blobinfo) + assert.NotNil(t, bi) +} + +func TestPutBlobHandleRequest(t *testing.T) { + req, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + blobinfo := &util.BlobInfo{} + blobinfo.Repository = "TestPutBlobHandleRequest/latest" + + bi := NewPutBlobInterceptor(blobinfo) + assert.NotNil(t, bi.HandleRequest(req)) +} + +func TestPutBlobHandleResponse(t *testing.T) { + con, err := redis.Dial( + "tcp", + fmt.Sprintf("%s:%d", getRedisHost(), 6379), + redis.DialConnectTimeout(30*time.Second), + redis.DialReadTimeout(time.Minute+10*time.Second), + redis.DialWriteTimeout(10*time.Second), + ) + assert.Nil(t, err) + defer con.Close() + + req, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + blobInfo := util.BlobInfo{ + Repository: "TestPutBlobHandleResponse/test", + Digest: "sha256:TestPutBlobHandleResponseabcdf12345678sdfefeg1246", + ContentType: "ContentType", + Size: 101, + Exist: false, + } + + rw := httptest.NewRecorder() + customResW := util.CustomResponseWriter{ResponseWriter: rw} + customResW.WriteHeader(201) + + lock, err := tryLockBlob(con, &blobInfo) + assert.Nil(t, err) + blobInfo.DigestLock = lock + + *req = *(req.WithContext(context.WithValue(req.Context(), util.BBInfokKey, &blobInfo))) + + bi := NewPutBlobInterceptor(&blobInfo) + assert.NotNil(t, bi) + + bi.HandleResponse(customResW, req) +} diff --git a/src/core/middlewares/sizequota/putmanifest.go b/src/core/middlewares/sizequota/putmanifest.go new file mode 100644 index 000000000..76d87044a --- /dev/null +++ b/src/core/middlewares/sizequota/putmanifest.go @@ -0,0 +1,102 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizequota + +import ( + "bytes" + "context" + "fmt" + "github.com/docker/distribution" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/util" + "io/ioutil" + "net/http" + "strings" +) + +// PutManifestInterceptor ... +type PutManifestInterceptor struct { + blobInfo *util.BlobInfo + mfInfo *util.MfInfo +} + +// NewPutManifestInterceptor ... +func NewPutManifestInterceptor(blobInfo *util.BlobInfo, mfInfo *util.MfInfo) *PutManifestInterceptor { + return &PutManifestInterceptor{ + blobInfo: blobInfo, + mfInfo: mfInfo, + } +} + +// HandleRequest ... +func (pmi *PutManifestInterceptor) HandleRequest(req *http.Request) error { + mediaType := req.Header.Get("Content-Type") + if mediaType == schema1.MediaTypeManifest || + mediaType == schema1.MediaTypeSignedManifest || + mediaType == schema2.MediaTypeManifest { + + con, err := util.GetRegRedisCon() + if err != nil { + log.Infof("failed to get registry redis connection, %v", err) + return err + } + + data, err := ioutil.ReadAll(req.Body) + if err != nil { + log.Warningf("Error occurred when to copy manifest body %v", err) + return err + } + req.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + manifest, desc, err := distribution.UnmarshalManifest(mediaType, data) + if err != nil { + log.Warningf("Error occurred when to Unmarshal Manifest %v", err) + return err + } + projectID, err := util.GetProjectID(strings.Split(pmi.mfInfo.Repository, "/")[0]) + if err != nil { + log.Warningf("Error occurred when to get project ID %v", err) + return err + } + + pmi.mfInfo.ProjectID = projectID + pmi.mfInfo.Refrerence = manifest.References() + pmi.mfInfo.Digest = desc.Digest.String() + pmi.blobInfo.ProjectID = projectID + pmi.blobInfo.Digest = desc.Digest.String() + pmi.blobInfo.Size = desc.Size + pmi.blobInfo.ContentType = mediaType + + if err := requireQuota(con, pmi.blobInfo); err != nil { + return err + } + + *req = *(req.WithContext(context.WithValue(req.Context(), util.MFInfokKey, pmi.mfInfo))) + *req = *(req.WithContext(context.WithValue(req.Context(), util.BBInfokKey, pmi.blobInfo))) + + return nil + } + + return fmt.Errorf("unsupported content type for manifest: %s", mediaType) +} + +// HandleResponse ... +func (pmi *PutManifestInterceptor) HandleResponse(rw util.CustomResponseWriter, req *http.Request) { + if err := HandleBlobCommon(rw, req); err != nil { + log.Error(err) + return + } +} diff --git a/src/core/middlewares/sizequota/putmanifest_test.go b/src/core/middlewares/sizequota/putmanifest_test.go new file mode 100644 index 000000000..dc6b91098 --- /dev/null +++ b/src/core/middlewares/sizequota/putmanifest_test.go @@ -0,0 +1,92 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizequota + +import ( + "context" + "fmt" + "github.com/garyburd/redigo/redis" + "github.com/goharbor/harbor/src/core/middlewares/util" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestNewPutManifestInterceptor(t *testing.T) { + blobinfo := &util.BlobInfo{} + blobinfo.Repository = "TestNewPutManifestInterceptor/latest" + + mfinfo := &util.MfInfo{ + Repository: "TestNewPutManifestInterceptor", + } + + mi := NewPutManifestInterceptor(blobinfo, mfinfo) + assert.NotNil(t, mi) +} + +func TestPutManifestHandleRequest(t *testing.T) { + req, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + blobinfo := &util.BlobInfo{} + blobinfo.Repository = "TestPutManifestHandleRequest/latest" + + mfinfo := &util.MfInfo{ + Repository: "TestPutManifestHandleRequest", + } + + mi := NewPutManifestInterceptor(blobinfo, mfinfo) + assert.NotNil(t, mi.HandleRequest(req)) +} + +func TestPutManifestHandleResponse(t *testing.T) { + con, err := redis.Dial( + "tcp", + fmt.Sprintf("%s:%d", getRedisHost(), 6379), + redis.DialConnectTimeout(30*time.Second), + redis.DialReadTimeout(time.Minute+10*time.Second), + redis.DialWriteTimeout(10*time.Second), + ) + assert.Nil(t, err) + defer con.Close() + + req, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + blobInfo := util.BlobInfo{ + Repository: "TestPutManifestandleResponse/test", + Digest: "sha256:TestPutManifestandleResponseabcdf12345678sdfefeg1246", + ContentType: "ContentType", + Size: 101, + Exist: false, + } + + mfinfo := util.MfInfo{ + Repository: "TestPutManifestandleResponse", + } + + rw := httptest.NewRecorder() + customResW := util.CustomResponseWriter{ResponseWriter: rw} + customResW.WriteHeader(201) + + lock, err := tryLockBlob(con, &blobInfo) + assert.Nil(t, err) + blobInfo.DigestLock = lock + + *req = *(req.WithContext(context.WithValue(req.Context(), util.BBInfokKey, &blobInfo))) + + bi := NewPutManifestInterceptor(&blobInfo, &mfinfo) + assert.NotNil(t, bi) + + bi.HandleResponse(customResW, req) +} diff --git a/src/core/middlewares/url/handler.go b/src/core/middlewares/url/handler.go new file mode 100644 index 000000000..07e1a0f3f --- /dev/null +++ b/src/core/middlewares/url/handler.go @@ -0,0 +1,74 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package url + +import ( + "context" + "fmt" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/util" + coreutils "github.com/goharbor/harbor/src/core/utils" + "net/http" + "strings" +) + +type urlHandler struct { + next http.Handler +} + +// New ... +func New(next http.Handler) http.Handler { + return &urlHandler{ + next: next, + } +} + +// ServeHTTP ... +func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + log.Debugf("in url handler, path: %s", req.URL.Path) + flag, repository, reference := util.MatchPullManifest(req) + if flag { + components := strings.SplitN(repository, "/", 2) + if len(components) < 2 { + http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest) + return + } + + client, err := coreutils.NewRepositoryClientForUI(util.TokenUsername, repository) + if err != nil { + log.Errorf("Error creating repository Client: %v", err) + http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError) + return + } + digest, _, err := client.ManifestExist(reference) + if err != nil { + log.Errorf("Failed to get digest for reference: %s, error: %v", reference, err) + http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError) + return + } + + img := util.ImageInfo{ + Repository: repository, + Reference: reference, + ProjectName: components[0], + Digest: digest, + } + + log.Debugf("image info of the request: %#v", img) + ctx := context.WithValue(req.Context(), util.ImageInfoCtxKey, img) + req = req.WithContext(ctx) + } + uh.next.ServeHTTP(rw, req) +} diff --git a/src/core/middlewares/util/reginteceptor.go b/src/core/middlewares/util/reginteceptor.go new file mode 100644 index 000000000..902b66f0a --- /dev/null +++ b/src/core/middlewares/util/reginteceptor.go @@ -0,0 +1,28 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "net/http" +) + +// RegInterceptor ... +type RegInterceptor interface { + // HandleRequest ... + HandleRequest(req *http.Request) error + + // HandleResponse won't return any error + HandleResponse(rw CustomResponseWriter, req *http.Request) +} diff --git a/src/core/middlewares/util/response.go b/src/core/middlewares/util/response.go new file mode 100644 index 000000000..48e3f0cda --- /dev/null +++ b/src/core/middlewares/util/response.go @@ -0,0 +1,59 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "net/http" +) + +// CustomResponseWriter write the response code into the status +type CustomResponseWriter struct { + http.ResponseWriter + status int + wroteHeader bool +} + +// NewCustomResponseWriter ... +func NewCustomResponseWriter(w http.ResponseWriter) *CustomResponseWriter { + return &CustomResponseWriter{ResponseWriter: w} +} + +// Status ... +func (w *CustomResponseWriter) Status() int { + return w.status +} + +// Header ... +func (w CustomResponseWriter) Header() http.Header { + return w.ResponseWriter.Header() +} + +// Write ... +func (w *CustomResponseWriter) Write(p []byte) (n int, err error) { + if !w.wroteHeader { + w.WriteHeader(http.StatusOK) + } + return w.ResponseWriter.Write(p) +} + +// WriteHeader ... +func (w *CustomResponseWriter) WriteHeader(code int) { + w.ResponseWriter.WriteHeader(code) + if w.wroteHeader { + return + } + w.status = code + w.wroteHeader = true +} diff --git a/src/core/middlewares/util/response_test.go b/src/core/middlewares/util/response_test.go new file mode 100644 index 000000000..40ec59c4e --- /dev/null +++ b/src/core/middlewares/util/response_test.go @@ -0,0 +1,29 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCustomResponseWriter(t *testing.T) { + rw := httptest.NewRecorder() + customResW := CustomResponseWriter{ResponseWriter: rw} + customResW.WriteHeader(501) + assert.Equal(t, customResW.Status(), 501) +} diff --git a/src/core/middlewares/util/util.go b/src/core/middlewares/util/util.go new file mode 100644 index 000000000..8e1df00e2 --- /dev/null +++ b/src/core/middlewares/util/util.go @@ -0,0 +1,377 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/docker/distribution" + "github.com/garyburd/redigo/redis" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota" + "github.com/goharbor/harbor/src/common/utils/clair" + "github.com/goharbor/harbor/src/common/utils/log" + common_redis "github.com/goharbor/harbor/src/common/utils/redis" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/promgr" + "github.com/goharbor/harbor/src/pkg/scan/whitelist" + "net/http" + "net/http/httptest" + "regexp" + "strconv" + "strings" + "time" +) + +type contextKey string + +// ErrRequireQuota ... +var ErrRequireQuota = errors.New("cannot get quota on project for request") + +const ( + manifestURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})` + blobURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)blobs/uploads/` + // ImageInfoCtxKey the context key for image information + ImageInfoCtxKey = contextKey("ImageInfo") + // TokenUsername ... + // TODO: temp solution, remove after vmware/harbor#2242 is resolved. + TokenUsername = "harbor-core" + // MFInfokKey the context key for image tag redis lock + MFInfokKey = contextKey("ManifestInfo") + // BBInfokKey the context key for image tag redis lock + BBInfokKey = contextKey("BlobInfo") + + // DialConnectionTimeout ... + DialConnectionTimeout = 30 * time.Second + // DialReadTimeout ... + DialReadTimeout = time.Minute + 10*time.Second + // DialWriteTimeout ... + DialWriteTimeout = 10 * time.Second +) + +// ImageInfo ... +type ImageInfo struct { + Repository string + Reference string + ProjectName string + Digest string +} + +// BlobInfo ... +type BlobInfo struct { + UUID string + ProjectID int64 + ContentType string + Size int64 + Repository string + Tag string + + // Exist is to index the existing of the manifest in DB. If false, it's an new image for uploading. + Exist bool + + Digest string + DigestLock *common_redis.Mutex + // Quota is the resource applied for the manifest upload request. + Quota *quota.ResourceList +} + +// MfInfo ... +type MfInfo struct { + // basic information of a manifest + ProjectID int64 + Repository string + Tag string + Digest string + + // Exist is to index the existing of the manifest in DB. If false, it's an new image for uploading. + Exist bool + // DigestChanged true means the manifest exists but digest is changed. + // Probably it's a new image with existing repo/tag name or overwrite. + DigestChanged bool + + // used to block multiple push on same image. + TagLock *common_redis.Mutex + Refrerence []distribution.Descriptor + + // Quota is the resource applied for the manifest upload request. + Quota *quota.ResourceList +} + +// JSONError wraps a concrete Code and Message, it's readable for docker deamon. +type JSONError struct { + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` + Detail string `json:"detail,omitempty"` +} + +// MarshalError ... +func MarshalError(code, msg string) string { + var tmpErrs struct { + Errors []JSONError `json:"errors,omitempty"` + } + tmpErrs.Errors = append(tmpErrs.Errors, JSONError{ + Code: code, + Message: msg, + Detail: msg, + }) + str, err := json.Marshal(tmpErrs) + if err != nil { + log.Debugf("failed to marshal json error, %v", err) + return msg + } + return string(str) +} + +// MatchManifestURL ... +func MatchManifestURL(req *http.Request) (bool, string, string) { + re, err := regexp.Compile(manifestURLPattern) + if err != nil { + log.Errorf("error to match manifest url, %v", err) + return false, "", "" + } + s := re.FindStringSubmatch(req.URL.Path) + if len(s) == 3 { + s[1] = strings.TrimSuffix(s[1], "/") + return true, s[1], s[2] + } + return false, "", "" +} + +// MatchPutBlobURL ... +func MatchPutBlobURL(req *http.Request) (bool, string) { + if req.Method != http.MethodPut { + return false, "" + } + re, err := regexp.Compile(blobURLPattern) + if err != nil { + log.Errorf("error to match put blob url, %v", err) + return false, "" + } + s := re.FindStringSubmatch(req.URL.Path) + if len(s) == 2 { + s[1] = strings.TrimSuffix(s[1], "/") + return true, s[1] + } + return false, "" +} + +// MatchPatchBlobURL ... +func MatchPatchBlobURL(req *http.Request) (bool, string) { + if req.Method != http.MethodPatch { + return false, "" + } + re, err := regexp.Compile(blobURLPattern) + if err != nil { + log.Errorf("error to match put blob url, %v", err) + return false, "" + } + s := re.FindStringSubmatch(req.URL.Path) + if len(s) == 2 { + s[1] = strings.TrimSuffix(s[1], "/") + return true, s[1] + } + return false, "" +} + +// MatchPullManifest checks if the request looks like a request to pull manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values +func MatchPullManifest(req *http.Request) (bool, string, string) { + if req.Method != http.MethodGet { + return false, "", "" + } + return MatchManifestURL(req) +} + +// MatchPushManifest checks if the request looks like a request to push manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values +func MatchPushManifest(req *http.Request) (bool, string, string) { + if req.Method != http.MethodPut { + return false, "", "" + } + return MatchManifestURL(req) +} + +// MatchMountBlobURL POST /v2//blobs/uploads/?mount=&from= +// If match, will return repo, mount and from as the 2nd, 3th and 4th. +func MatchMountBlobURL(req *http.Request) (bool, string, string, string) { + if req.Method != http.MethodPost { + return false, "", "", "" + } + re, err := regexp.Compile(blobURLPattern) + if err != nil { + log.Errorf("error to match post blob url, %v", err) + return false, "", "", "" + } + s := re.FindStringSubmatch(req.URL.Path) + if len(s) == 2 { + s[1] = strings.TrimSuffix(s[1], "/") + mount := req.FormValue("mount") + if mount == "" { + return false, "", "", "" + } + from := req.FormValue("from") + if from == "" { + return false, "", "", "" + } + return true, s[1], mount, from + } + return false, "", "", "" +} + +// CopyResp ... +func CopyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) { + for k, v := range rec.Header() { + rw.Header()[k] = v + } + rw.WriteHeader(rec.Result().StatusCode) + rw.Write(rec.Body.Bytes()) +} + +// PolicyChecker checks the policy of a project by project name, to determine if it's needed to check the image's status under this project. +type PolicyChecker interface { + // contentTrustEnabled returns whether a project has enabled content trust. + ContentTrustEnabled(name string) bool + // vulnerablePolicy returns whether a project has enabled vulnerable, and the project's severity. + VulnerablePolicy(name string) (bool, models.Severity, models.CVEWhitelist) +} + +// PmsPolicyChecker ... +type PmsPolicyChecker struct { + pm promgr.ProjectManager +} + +// ContentTrustEnabled ... +func (pc PmsPolicyChecker) ContentTrustEnabled(name string) bool { + project, err := pc.pm.Get(name) + if err != nil { + log.Errorf("Unexpected error when getting the project, error: %v", err) + return true + } + return project.ContentTrustEnabled() +} + +// VulnerablePolicy ... +func (pc PmsPolicyChecker) VulnerablePolicy(name string) (bool, models.Severity, models.CVEWhitelist) { + project, err := pc.pm.Get(name) + wl := models.CVEWhitelist{} + if err != nil { + log.Errorf("Unexpected error when getting the project, error: %v", err) + return true, models.SevUnknown, wl + } + mgr := whitelist.NewDefaultManager() + if project.ReuseSysCVEWhitelist() { + w, err := mgr.GetSys() + if err != nil { + return project.VulPrevented(), clair.ParseClairSev(project.Severity()), wl + } + wl = *w + } else { + w, err := mgr.Get(project.ProjectID) + if err != nil { + return project.VulPrevented(), clair.ParseClairSev(project.Severity()), wl + } + wl = *w + } + return project.VulPrevented(), clair.ParseClairSev(project.Severity()), wl + +} + +// NewPMSPolicyChecker returns an instance of an pmsPolicyChecker +func NewPMSPolicyChecker(pm promgr.ProjectManager) PolicyChecker { + return &PmsPolicyChecker{ + pm: pm, + } +} + +// GetPolicyChecker ... +func GetPolicyChecker() PolicyChecker { + return NewPMSPolicyChecker(config.GlobalProjectMgr) +} + +// TryRequireQuota ... +func TryRequireQuota(projectID int64, quotaRes *quota.ResourceList) error { + quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10)) + if err != nil { + log.Errorf("Error occurred when to new quota manager %v", err) + return err + } + if err := quotaMgr.AddResources(*quotaRes); err != nil { + log.Errorf("cannot get quota for the project resource: %d, err: %v", projectID, err) + return ErrRequireQuota + } + return nil +} + +// TryFreeQuota used to release resource for failure case +func TryFreeQuota(projectID int64, qres *quota.ResourceList) bool { + quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10)) + if err != nil { + log.Errorf("Error occurred when to new quota manager %v", err) + return false + } + + if err := quotaMgr.SubtractResources(*qres); err != nil { + log.Errorf("cannot release quota for the project resource: %d, err: %v", projectID, err) + return false + } + return true +} + +// GetBlobSize blob size with UUID in redis +func GetBlobSize(conn redis.Conn, uuid string) (int64, error) { + exists, err := redis.Int(conn.Do("EXISTS", uuid)) + if err != nil { + return 0, err + } + if exists == 1 { + size, err := redis.Int64(conn.Do("GET", uuid)) + if err != nil { + return 0, err + } + return size, nil + } + return 0, nil +} + +// SetBunkSize sets the temp size for blob bunk with its uuid. +func SetBunkSize(conn redis.Conn, uuid string, size int64) (bool, error) { + setRes, err := redis.String(conn.Do("SET", uuid, size)) + if err != nil { + return false, err + } + return setRes == "OK", nil +} + +// GetProjectID ... +func GetProjectID(name string) (int64, error) { + project, err := dao.GetProjectByName(name) + if err != nil { + return 0, err + } + if project != nil { + return project.ProjectID, nil + } + return 0, fmt.Errorf("project %s is not found", name) +} + +// GetRegRedisCon ... +func GetRegRedisCon() (redis.Conn, error) { + return redis.DialURL( + config.GetRedisOfRegURL(), + redis.DialConnectTimeout(DialConnectionTimeout), + redis.DialReadTimeout(DialReadTimeout), + redis.DialWriteTimeout(DialWriteTimeout), + ) +} diff --git a/src/core/proxy/interceptor_test.go b/src/core/middlewares/util/util_test.go similarity index 58% rename from src/core/proxy/interceptor_test.go rename to src/core/middlewares/util/util_test.go index db9912602..736c81a05 100644 --- a/src/core/proxy/interceptor_test.go +++ b/src/core/middlewares/util/util_test.go @@ -1,7 +1,22 @@ -package proxy +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util import ( "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" notarytest "github.com/goharbor/harbor/src/common/utils/notary/test" testutils "github.com/goharbor/harbor/src/common/utils/test" @@ -9,22 +24,28 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "fmt" + "github.com/garyburd/redigo/redis" + "github.com/goharbor/harbor/src/common/quota" "net/http" "net/http/httptest" "os" "testing" + "time" ) var endpoint = "10.117.4.142" var notaryServer *httptest.Server +const testingRedisHost = "REDIS_HOST" + var admiralEndpoint = "http://127.0.0.1:8282" var token = "" func TestMain(m *testing.M) { + testutils.InitDatabaseFromEnv() notaryServer = notarytest.NewNotaryServer(endpoint) defer notaryServer.Close() - NotaryEndpoint = notaryServer.URL var defaultConfig = map[string]interface{}{ common.ExtEndpoint: "https://" + endpoint, common.WithNotary: true, @@ -78,6 +99,56 @@ func TestMatchPullManifest(t *testing.T) { assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7) } +func TestMatchPutBlob(t *testing.T) { + assert := assert.New(t) + req1, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/blobs/uploads/67bb4d9b-4dab-4bbe-b726-2e39322b8303?_state=7W3kWkgdr3fTW", nil) + res1, repo1 := MatchPutBlobURL(req1) + assert.True(res1, "%s %v is not a request to put blob", req1.Method, req1.URL) + assert.Equal("library/ubuntu", repo1) + + req2, _ := http.NewRequest("PATCH", "http://127.0.0.1:5000/v2/library/blobs/uploads/67bb4d9b-4dab-4bbe-b726-2e39322b8303?_state=7W3kWkgdr3fTW", nil) + res2, _ := MatchPutBlobURL(req2) + assert.False(res2, "%s %v is a request to put blob", req2.Method, req2.URL) + + req3, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/manifest/67bb4d9b-4dab-4bbe-b726-2e39322b8303?_state=7W3kWkgdr3fTW", nil) + res3, _ := MatchPutBlobURL(req3) + assert.False(res3, "%s %v is not a request to put blob", req3.Method, req3.URL) +} + +func TestMatchMountBlobURL(t *testing.T) { + assert := assert.New(t) + req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/blobs/uploads/?mount=digtest123&from=testrepo", nil) + res1, repo1, mount, from := MatchMountBlobURL(req1) + assert.True(res1, "%s %v is not a request to mount blob", req1.Method, req1.URL) + assert.Equal("library/ubuntu", repo1) + assert.Equal("digtest123", mount) + assert.Equal("testrepo", from) + + req2, _ := http.NewRequest("PATCH", "http://127.0.0.1:5000/v2/library/ubuntu/blobs/uploads/?mount=digtest123&from=testrepo", nil) + res2, _, _, _ := MatchMountBlobURL(req2) + assert.False(res2, "%s %v is a request to mount blob", req2.Method, req2.URL) + + req3, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/blobs/uploads/?mount=digtest123&from=testrepo", nil) + res3, _, _, _ := MatchMountBlobURL(req3) + assert.False(res3, "%s %v is not a request to put blob", req3.Method, req3.URL) +} + +func TestPatchBlobURL(t *testing.T) { + assert := assert.New(t) + req1, _ := http.NewRequest("PATCH", "http://127.0.0.1:5000/v2/library/ubuntu/blobs/uploads/1234-1234-abcd", nil) + res1, repo1 := MatchPatchBlobURL(req1) + assert.True(res1, "%s %v is not a request to patch blob", req1.Method, req1.URL) + assert.Equal("library/ubuntu", repo1) + + req2, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/blobs/uploads/1234-1234-abcd", nil) + res2, _ := MatchPatchBlobURL(req2) + assert.False(res2, "%s %v is a request to patch blob", req2.Method, req2.URL) + + req3, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/blobs/uploads/?mount=digtest123&from=testrepo", nil) + res3, _ := MatchPatchBlobURL(req3) + assert.False(res3, "%s %v is not a request to patch blob", req3.Method, req3.URL) +} + func TestMatchPushManifest(t *testing.T) { assert := assert.New(t) req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) @@ -125,22 +196,6 @@ func TestMatchPushManifest(t *testing.T) { assert.Equal("14.04", tag8) } -func TestMatchListRepos(t *testing.T) { - assert := assert.New(t) - req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/_catalog", nil) - res1 := MatchListRepos(req1) - assert.False(res1, "%s %v is not a request to list repos", req1.Method, req1.URL) - - req2, _ := http.NewRequest("GET", "http://127.0.0.1:5000/v2/_catalog", nil) - res2 := MatchListRepos(req2) - assert.True(res2, "%s %v is a request to list repos", req2.Method, req2.URL) - - req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/_catalog", nil) - res3 := MatchListRepos(req3) - assert.False(res3, "%s %v is not a request to pull manifest", req3.Method, req3.URL) - -} - func TestPMSPolicyChecker(t *testing.T) { var defaultConfigAdmiral = map[string]interface{}{ common.ExtEndpoint: "https://" + endpoint, @@ -157,7 +212,6 @@ func TestPMSPolicyChecker(t *testing.T) { if err := config.Init(); err != nil { panic(err) } - testutils.InitDatabaseFromEnv() config.Upload(defaultConfigAdmiral) @@ -179,50 +233,110 @@ func TestPMSPolicyChecker(t *testing.T) { } }(id) - contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_sev_low") + contentTrustFlag := GetPolicyChecker().ContentTrustEnabled("project_for_test_get_sev_low") assert.True(t, contentTrustFlag) - projectVulnerableEnabled, projectVulnerableSeverity, wl := getPolicyChecker().vulnerablePolicy("project_for_test_get_sev_low") + projectVulnerableEnabled, projectVulnerableSeverity, wl := GetPolicyChecker().VulnerablePolicy("project_for_test_get_sev_low") assert.True(t, projectVulnerableEnabled) assert.Equal(t, projectVulnerableSeverity, models.SevLow) assert.Empty(t, wl.Items) } -func TestMatchNotaryDigest(t *testing.T) { - assert := assert.New(t) - // The data from common/utils/notary/helper_test.go - img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo", "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"} - img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo", "sha256:12345678"} - - res1, err := matchNotaryDigest(img1) - assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1) - assert.True(res1) - - res2, err := matchNotaryDigest(img2) - assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2) - assert.False(res2) -} - func TestCopyResp(t *testing.T) { assert := assert.New(t) rec1 := httptest.NewRecorder() rec2 := httptest.NewRecorder() rec1.Header().Set("X-Test", "mytest") rec1.WriteHeader(418) - copyResp(rec1, rec2) + CopyResp(rec1, rec2) assert.Equal(418, rec2.Result().StatusCode) assert.Equal("mytest", rec2.Header().Get("X-Test")) } func TestMarshalError(t *testing.T) { assert := assert.New(t) - js1 := marshalError("PROJECT_POLICY_VIOLATION", "Not Found") + js1 := MarshalError("PROJECT_POLICY_VIOLATION", "Not Found") assert.Equal("{\"errors\":[{\"code\":\"PROJECT_POLICY_VIOLATION\",\"message\":\"Not Found\",\"detail\":\"Not Found\"}]}", js1) - js2 := marshalError("DENIED", "The action is denied") + js2 := MarshalError("DENIED", "The action is denied") assert.Equal("{\"errors\":[{\"code\":\"DENIED\",\"message\":\"The action is denied\",\"detail\":\"The action is denied\"}]}", js2) } -func TestIsDigest(t *testing.T) { - assert := assert.New(t) - assert.False(isDigest("latest")) - assert.True(isDigest("sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7")) +func TestTryRequireQuota(t *testing.T) { + quotaRes := "a.ResourceList{ + quota.ResourceStorage: 100, + } + err := TryRequireQuota(1, quotaRes) + assert.Nil(t, err) +} + +func TestTryFreeQuota(t *testing.T) { + quotaRes := "a.ResourceList{ + quota.ResourceStorage: 1, + } + success := TryFreeQuota(1, quotaRes) + assert.True(t, success) +} + +func TestGetBlobSize(t *testing.T) { + con, err := redis.Dial( + "tcp", + fmt.Sprintf("%s:%d", getRedisHost(), 6379), + redis.DialConnectTimeout(30*time.Second), + redis.DialReadTimeout(time.Minute+10*time.Second), + redis.DialWriteTimeout(10*time.Second), + ) + assert.Nil(t, err) + defer con.Close() + + size, err := GetBlobSize(con, "test-TestGetBlobSize") + assert.Nil(t, err) + assert.Equal(t, size, int64(0)) +} + +func TestSetBunkSize(t *testing.T) { + con, err := redis.Dial( + "tcp", + fmt.Sprintf("%s:%d", getRedisHost(), 6379), + redis.DialConnectTimeout(30*time.Second), + redis.DialReadTimeout(time.Minute+10*time.Second), + redis.DialWriteTimeout(10*time.Second), + ) + assert.Nil(t, err) + defer con.Close() + + size, err := GetBlobSize(con, "TestSetBunkSize") + assert.Nil(t, err) + assert.Equal(t, size, int64(0)) + + _, err = SetBunkSize(con, "TestSetBunkSize", 123) + assert.Nil(t, err) + + size1, err := GetBlobSize(con, "TestSetBunkSize") + assert.Nil(t, err) + assert.Equal(t, size1, int64(123)) +} + +func TestGetProjectID(t *testing.T) { + name := "project_for_TestGetProjectID" + project := models.Project{ + OwnerID: 1, + Name: name, + } + + id, err := dao.AddProject(project) + if err != nil { + t.Fatalf("failed to add project: %v", err) + } + + idget, err := GetProjectID(name) + assert.Nil(t, err) + assert.Equal(t, id, idget) +} + +func getRedisHost() string { + redisHost := os.Getenv(testingRedisHost) + if redisHost == "" { + redisHost = "127.0.0.1" // for local test + } + + return redisHost } diff --git a/src/core/middlewares/vulnerable/handler.go b/src/core/middlewares/vulnerable/handler.go new file mode 100644 index 000000000..67d1b97ce --- /dev/null +++ b/src/core/middlewares/vulnerable/handler.go @@ -0,0 +1,80 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vulnerable + +import ( + "fmt" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/middlewares/util" + "github.com/goharbor/harbor/src/pkg/scan" + "net/http" +) + +type vulnerableHandler struct { + next http.Handler +} + +// New ... +func New(next http.Handler) http.Handler { + return &vulnerableHandler{ + next: next, + } +} + +// ServeHTTP ... +func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + imgRaw := req.Context().Value(util.ImageInfoCtxKey) + if imgRaw == nil || !config.WithClair() { + vh.next.ServeHTTP(rw, req) + return + } + img, _ := req.Context().Value(util.ImageInfoCtxKey).(util.ImageInfo) + if img.Digest == "" { + vh.next.ServeHTTP(rw, req) + return + } + projectVulnerableEnabled, projectVulnerableSeverity, wl := util.GetPolicyChecker().VulnerablePolicy(img.ProjectName) + if !projectVulnerableEnabled { + vh.next.ServeHTTP(rw, req) + return + } + vl, err := scan.VulnListByDigest(img.Digest) + if err != nil { + log.Errorf("Failed to get the vulnerability list, error: %v", err) + http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "Failed to get vulnerabilities."), http.StatusPreconditionFailed) + return + } + filtered := vl.ApplyWhitelist(wl) + msg := vh.filterMsg(img, filtered) + log.Info(msg) + if int(vl.Severity()) >= int(projectVulnerableSeverity) { + log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", vl.Severity(), projectVulnerableSeverity) + http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", vl.Severity(), projectVulnerableSeverity)), http.StatusPreconditionFailed) + return + } + vh.next.ServeHTTP(rw, req) +} + +func (vh vulnerableHandler) filterMsg(img util.ImageInfo, filtered scan.VulnerabilityList) string { + filterMsg := fmt.Sprintf("Image: %s/%s:%s, digest: %s, vulnerabilities fitered by whitelist:", img.ProjectName, img.Repository, img.Reference, img.Digest) + if len(filtered) == 0 { + filterMsg = fmt.Sprintf("%s none.", filterMsg) + } + for _, v := range filtered { + filterMsg = fmt.Sprintf("%s ID: %s, severity: %s;", filterMsg, v.ID, v.Severity) + } + return filterMsg +} diff --git a/src/core/proxy/interceptors.go b/src/core/proxy/interceptors.go deleted file mode 100644 index 6e28aaa6e..000000000 --- a/src/core/proxy/interceptors.go +++ /dev/null @@ -1,420 +0,0 @@ -package proxy - -import ( - "encoding/json" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils/clair" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/common/utils/notary" - "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/core/promgr" - coreutils "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/pkg/scan" - "github.com/goharbor/harbor/src/pkg/scan/whitelist" - - "context" - "fmt" - "net/http" - "net/http/httptest" - "regexp" - "strconv" - "strings" -) - -type contextKey string - -const ( - manifestURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})` - catalogURLPattern = `/v2/_catalog` - imageInfoCtxKey = contextKey("ImageInfo") - // TODO: temp solution, remove after vmware/harbor#2242 is resolved. - tokenUsername = "harbor-core" -) - -// Record the docker deamon raw response. -var rec *httptest.ResponseRecorder - -// NotaryEndpoint , exported for testing. -var NotaryEndpoint = "" - -// MatchPullManifest checks if the request looks like a request to pull manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values -func MatchPullManifest(req *http.Request) (bool, string, string) { - // TODO: add user agent check. - if req.Method != http.MethodGet { - return false, "", "" - } - return matchManifestURL(req) -} - -// MatchPushManifest checks if the request looks like a request to push manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values -func MatchPushManifest(req *http.Request) (bool, string, string) { - if req.Method != http.MethodPut { - return false, "", "" - } - return matchManifestURL(req) -} - -func matchManifestURL(req *http.Request) (bool, string, string) { - re := regexp.MustCompile(manifestURLPattern) - s := re.FindStringSubmatch(req.URL.Path) - if len(s) == 3 { - s[1] = strings.TrimSuffix(s[1], "/") - return true, s[1], s[2] - } - return false, "", "" -} - -// MatchListRepos checks if the request looks like a request to list repositories. -func MatchListRepos(req *http.Request) bool { - if req.Method != http.MethodGet { - return false - } - re := regexp.MustCompile(catalogURLPattern) - s := re.FindStringSubmatch(req.URL.Path) - if len(s) == 1 { - return true - } - return false -} - -// policyChecker checks the policy of a project by project name, to determine if it's needed to check the image's status under this project. -type policyChecker interface { - // contentTrustEnabled returns whether a project has enabled content trust. - contentTrustEnabled(name string) bool - // vulnerablePolicy returns whether a project has enabled vulnerable, and the project's severity. - vulnerablePolicy(name string) (bool, models.Severity, models.CVEWhitelist) -} - -type pmsPolicyChecker struct { - pm promgr.ProjectManager -} - -func (pc pmsPolicyChecker) contentTrustEnabled(name string) bool { - project, err := pc.pm.Get(name) - if err != nil { - log.Errorf("Unexpected error when getting the project, error: %v", err) - return true - } - return project.ContentTrustEnabled() -} -func (pc pmsPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity, models.CVEWhitelist) { - project, err := pc.pm.Get(name) - wl := models.CVEWhitelist{} - if err != nil { - log.Errorf("Unexpected error when getting the project, error: %v", err) - return true, models.SevUnknown, wl - } - mgr := whitelist.NewDefaultManager() - if project.ReuseSysCVEWhitelist() { - w, err := mgr.GetSys() - if err != nil { - return project.VulPrevented(), clair.ParseClairSev(project.Severity()), wl - } - wl = *w - } else { - w, err := mgr.Get(project.ProjectID) - if err != nil { - return project.VulPrevented(), clair.ParseClairSev(project.Severity()), wl - } - wl = *w - } - return project.VulPrevented(), clair.ParseClairSev(project.Severity()), wl -} - -// newPMSPolicyChecker returns an instance of an pmsPolicyChecker -func newPMSPolicyChecker(pm promgr.ProjectManager) policyChecker { - return &pmsPolicyChecker{ - pm: pm, - } -} - -func getPolicyChecker() policyChecker { - return newPMSPolicyChecker(config.GlobalProjectMgr) -} - -type imageInfo struct { - repository string - reference string - projectName string - digest string -} - -type urlHandler struct { - next http.Handler -} - -func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - log.Debugf("in url handler, path: %s", req.URL.Path) - flag, repository, reference := MatchPullManifest(req) - if flag { - components := strings.SplitN(repository, "/", 2) - if len(components) < 2 { - http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest) - return - } - - client, err := coreutils.NewRepositoryClientForUI(tokenUsername, repository) - if err != nil { - log.Errorf("Error creating repository Client: %v", err) - http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError) - return - } - digest, _, err := client.ManifestExist(reference) - if err != nil { - log.Errorf("Failed to get digest for reference: %s, error: %v", reference, err) - http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError) - return - } - - img := imageInfo{ - repository: repository, - reference: reference, - projectName: components[0], - digest: digest, - } - - log.Debugf("image info of the request: %#v", img) - ctx := context.WithValue(req.Context(), imageInfoCtxKey, img) - req = req.WithContext(ctx) - } - uh.next.ServeHTTP(rw, req) -} - -type readonlyHandler struct { - next http.Handler -} - -func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - if config.ReadOnly() { - if req.Method == http.MethodDelete || req.Method == http.MethodPost || req.Method == http.MethodPatch || req.Method == http.MethodPut { - log.Warningf("The request is prohibited in readonly mode, url is: %s", req.URL.Path) - http.Error(rw, marshalError("DENIED", "The system is in read only mode. Any modification is prohibited."), http.StatusForbidden) - return - } - } - rh.next.ServeHTTP(rw, req) -} - -type multipleManifestHandler struct { - next http.Handler -} - -// The handler is responsible for blocking request to upload manifest list by docker client, which is not supported so far by Harbor. -func (mh multipleManifestHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - match, _, _ := MatchPushManifest(req) - if match { - contentType := req.Header.Get("Content-type") - // application/vnd.docker.distribution.manifest.list.v2+json - if strings.Contains(contentType, "manifest.list.v2") { - log.Debugf("Content-type: %s is not supported, failing the response.", contentType) - http.Error(rw, marshalError("UNSUPPORTED_MEDIA_TYPE", "Manifest.list is not supported."), http.StatusUnsupportedMediaType) - return - } - } - mh.next.ServeHTTP(rw, req) -} - -type listReposHandler struct { - next http.Handler -} - -func (lrh listReposHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - listReposFlag := MatchListRepos(req) - if listReposFlag { - rec = httptest.NewRecorder() - lrh.next.ServeHTTP(rec, req) - if rec.Result().StatusCode != http.StatusOK { - copyResp(rec, rw) - return - } - var ctlg struct { - Repositories []string `json:"repositories"` - } - decoder := json.NewDecoder(rec.Body) - if err := decoder.Decode(&ctlg); err != nil { - log.Errorf("Decode repositories error: %v", err) - copyResp(rec, rw) - return - } - var entries []string - for repo := range ctlg.Repositories { - log.Debugf("the repo in the response %s", ctlg.Repositories[repo]) - exist := dao.RepositoryExists(ctlg.Repositories[repo]) - if exist { - entries = append(entries, ctlg.Repositories[repo]) - } - } - type Repos struct { - Repositories []string `json:"repositories"` - } - resp := &Repos{Repositories: entries} - respJSON, err := json.Marshal(resp) - if err != nil { - log.Errorf("Encode repositories error: %v", err) - copyResp(rec, rw) - return - } - - for k, v := range rec.Header() { - rw.Header()[k] = v - } - clen := len(respJSON) - rw.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(clen)) - rw.Write(respJSON) - return - } - lrh.next.ServeHTTP(rw, req) -} - -type contentTrustHandler struct { - next http.Handler -} - -func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - imgRaw := req.Context().Value(imageInfoCtxKey) - if imgRaw == nil || !config.WithNotary() { - cth.next.ServeHTTP(rw, req) - return - } - img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo) - if img.digest == "" { - cth.next.ServeHTTP(rw, req) - return - } - if !getPolicyChecker().contentTrustEnabled(img.projectName) { - cth.next.ServeHTTP(rw, req) - return - } - match, err := matchNotaryDigest(img) - if err != nil { - http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed in communication with Notary please check the log"), http.StatusInternalServerError) - return - } - if !match { - log.Debugf("digest mismatch, failing the response.") - http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "The image is not signed in Notary."), http.StatusPreconditionFailed) - return - } - cth.next.ServeHTTP(rw, req) -} - -type vulnerableHandler struct { - next http.Handler -} - -func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - imgRaw := req.Context().Value(imageInfoCtxKey) - if imgRaw == nil || !config.WithClair() { - vh.next.ServeHTTP(rw, req) - return - } - img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo) - if img.digest == "" { - vh.next.ServeHTTP(rw, req) - return - } - projectVulnerableEnabled, projectVulnerableSeverity, wl := getPolicyChecker().vulnerablePolicy(img.projectName) - if !projectVulnerableEnabled { - vh.next.ServeHTTP(rw, req) - return - } - vl, err := scan.VulnListByDigest(img.digest) - if err != nil { - log.Errorf("Failed to get the vulnerability list, error: %v", err) - http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get vulnerabilities."), http.StatusPreconditionFailed) - return - } - filtered := vl.ApplyWhitelist(wl) - msg := vh.filterMsg(img, filtered) - log.Info(msg) - if int(vl.Severity()) >= int(projectVulnerableSeverity) { - log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", vl.Severity(), projectVulnerableSeverity) - http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", vl.Severity(), projectVulnerableSeverity)), http.StatusPreconditionFailed) - return - } - vh.next.ServeHTTP(rw, req) -} - -func (vh vulnerableHandler) filterMsg(img imageInfo, filtered scan.VulnerabilityList) string { - filterMsg := fmt.Sprintf("Image: %s/%s:%s, digest: %s, vulnerabilities fitered by whitelist:", img.projectName, img.repository, img.reference, img.digest) - if len(filtered) == 0 { - filterMsg = fmt.Sprintf("%s none.", filterMsg) - } - for _, v := range filtered { - filterMsg = fmt.Sprintf("%s ID: %s, severity: %s;", filterMsg, v.ID, v.Severity) - } - return filterMsg -} - -func matchNotaryDigest(img imageInfo) (bool, error) { - if NotaryEndpoint == "" { - NotaryEndpoint = config.InternalNotaryEndpoint() - } - targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository) - if err != nil { - return false, err - } - for _, t := range targets { - if isDigest(img.reference) { - d, err := notary.DigestFromTarget(t) - if err != nil { - return false, err - } - if img.digest == d { - return true, nil - } - } else { - if t.Tag == img.reference { - log.Debugf("found reference: %s in notary, try to match digest.", img.reference) - d, err := notary.DigestFromTarget(t) - if err != nil { - return false, err - } - if img.digest == d { - return true, nil - } - } - } - } - log.Debugf("image: %#v, not found in notary", img) - return false, nil -} - -// A sha256 is a string with 64 characters. -func isDigest(ref string) bool { - return strings.HasPrefix(ref, "sha256:") && len(ref) == 71 -} - -func copyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) { - for k, v := range rec.Header() { - rw.Header()[k] = v - } - rw.WriteHeader(rec.Result().StatusCode) - rw.Write(rec.Body.Bytes()) -} - -func marshalError(code, msg string) string { - var tmpErrs struct { - Errors []JSONError `json:"errors,omitempty"` - } - tmpErrs.Errors = append(tmpErrs.Errors, JSONError{ - Code: code, - Message: msg, - Detail: msg, - }) - - str, err := json.Marshal(tmpErrs) - if err != nil { - log.Debugf("failed to marshal json error, %v", err) - return msg - } - return string(str) -} - -// JSONError wraps a concrete Code and Message, it's readable for docker deamon. -type JSONError struct { - Code string `json:"code,omitempty"` - Message string `json:"message,omitempty"` - Detail string `json:"detail,omitempty"` -} diff --git a/src/core/proxy/proxy.go b/src/core/proxy/proxy.go deleted file mode 100644 index bc0d7f44a..000000000 --- a/src/core/proxy/proxy.go +++ /dev/null @@ -1,56 +0,0 @@ -package proxy - -import ( - "github.com/goharbor/harbor/src/core/config" - - "fmt" - "net/http" - "net/http/httputil" - "net/url" -) - -// Proxy is the instance of the reverse proxy in this package. -var Proxy *httputil.ReverseProxy - -var handlers handlerChain - -type handlerChain struct { - head http.Handler -} - -// Init initialize the Proxy instance and handler chain. -func Init(urls ...string) error { - var err error - var registryURL string - if len(urls) > 1 { - return fmt.Errorf("the parm, urls should have only 0 or 1 elements") - } - if len(urls) == 0 { - registryURL, err = config.RegistryURL() - if err != nil { - return err - } - } else { - registryURL = urls[0] - } - targetURL, err := url.Parse(registryURL) - if err != nil { - return err - } - Proxy = httputil.NewSingleHostReverseProxy(targetURL) - handlers = handlerChain{ - head: readonlyHandler{ - next: urlHandler{ - next: multipleManifestHandler{ - next: listReposHandler{ - next: contentTrustHandler{ - next: vulnerableHandler{ - next: Proxy, - }}}}}}} - return nil -} - -// Handle handles the request. -func Handle(rw http.ResponseWriter, req *http.Request) { - handlers.head.ServeHTTP(rw, req) -} diff --git a/src/core/router.go b/src/core/router.go index f6cf911b1..3747bfc91 100644 --- a/src/core/router.go +++ b/src/core/router.go @@ -68,6 +68,7 @@ func initRouters() { beego.Router("/api/ping", &api.SystemInfoAPI{}, "get:Ping") beego.Router("/api/search", &api.SearchAPI{}) beego.Router("/api/projects/", &api.ProjectAPI{}, "get:List;post:Post") + beego.Router("/api/projects/:id([0-9]+)/summary", &api.ProjectAPI{}, "get:Summary") beego.Router("/api/projects/:id([0-9]+)/logs", &api.ProjectAPI{}, "get:Logs") beego.Router("/api/projects/:id([0-9]+)/_deletable", &api.ProjectAPI{}, "get:Deletable") beego.Router("/api/projects/:id([0-9]+)/metadatas/?:name", &api.MetadataAPI{}, "get:Get") @@ -77,6 +78,9 @@ func initRouters() { beego.Router("/api/projects/:pid([0-9]+)/robots", &api.RobotAPI{}, "post:Post;get:List") beego.Router("/api/projects/:pid([0-9]+)/robots/:id([0-9]+)", &api.RobotAPI{}, "get:Get;put:Put;delete:Delete") + beego.Router("/api/quotas", &api.QuotaAPI{}, "get:List") + beego.Router("/api/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put") + beego.Router("/api/repositories", &api.RepositoryAPI{}, "get:Get") beego.Router("/api/repositories/*", &api.RepositoryAPI{}, "delete:Delete;put:Put") beego.Router("/api/repositories/*/labels", &api.RepositoryLabelAPI{}, "get:GetOfRepository;post:AddToRepository") diff --git a/src/go.mod b/src/go.mod index c14aa5b4a..45cdd1f68 100644 --- a/src/go.mod +++ b/src/go.mod @@ -2,6 +2,8 @@ module github.com/goharbor/harbor/src go 1.12 +replace github.com/goharbor/harbor => ../ + require ( github.com/Knetic/govaluate v3.0.0+incompatible // indirect github.com/Masterminds/semver v1.4.2 @@ -44,8 +46,10 @@ require ( github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.6.2 + github.com/graph-gophers/dataloader v5.0.0+incompatible github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/jinzhu/gorm v1.9.8 // indirect + github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/lib/pq v1.1.0 @@ -54,6 +58,7 @@ require ( github.com/olekukonko/tablewriter v0.0.1 github.com/opencontainers/go-digest v1.0.0-rc0 github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/opentracing/opentracing-go v1.1.0 // indirect github.com/pkg/errors v0.8.1 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect github.com/prometheus/client_golang v0.9.4 // indirect diff --git a/src/go.sum b/src/go.sum index aa63848dd..0a7c3b720 100644 --- a/src/go.sum +++ b/src/go.sum @@ -145,6 +145,8 @@ github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/ github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/graph-gophers/dataloader v5.0.0+incompatible h1:R+yjsbrNq1Mo3aPG+Z/EKYrXrXXUNJHOgbRt+U6jOug= +github.com/graph-gophers/dataloader v5.0.0+incompatible/go.mod h1:jk4jk0c5ZISbKaMe8WsVopGB5/15GvGHMdMdPtwlRp4= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -169,6 +171,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da h1:5y58+OCjoHCYB8182mpf/dEsq0vwTKPOo4zGfH0xW9A= +github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da/go.mod h1:oLH0CmIaxCGXD67VKGR5AacGXZSMznlmeqM8RzPrcY8= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -211,6 +215,8 @@ github.com/opencontainers/go-digest v1.0.0-rc0 h1:YHPGfp+qlmg7loi376Jk5jNEgjgUUI github.com/opencontainers/go-digest v1.0.0-rc0/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= diff --git a/src/pkg/types/resources.go b/src/pkg/types/resources.go new file mode 100644 index 000000000..f30d91934 --- /dev/null +++ b/src/pkg/types/resources.go @@ -0,0 +1,115 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "encoding/json" +) + +const ( + // UNLIMITED unlimited resource value + UNLIMITED = -1 + + // ResourceCount count, in number + ResourceCount ResourceName = "count" + // ResourceStorage storage size, in bytes + ResourceStorage ResourceName = "storage" +) + +// ResourceName is the name identifying various resources in a ResourceList. +type ResourceName string + +// ResourceList is a set of (resource name, value) pairs. +type ResourceList map[ResourceName]int64 + +func (resources ResourceList) String() string { + bytes, _ := json.Marshal(resources) + return string(bytes) +} + +// NewResourceList returns resource list from string +func NewResourceList(s string) (ResourceList, error) { + var resources ResourceList + if err := json.Unmarshal([]byte(s), &resources); err != nil { + return nil, err + } + + return resources, nil +} + +// Equals returns true if the two lists are equivalent +func Equals(a ResourceList, b ResourceList) bool { + if len(a) != len(b) { + return false + } + + for key, value1 := range a { + value2, found := b[key] + if !found { + return false + } + if value1 != value2 { + return false + } + } + + return true +} + +// Add returns the result of a + b for each named resource +func Add(a ResourceList, b ResourceList) ResourceList { + result := ResourceList{} + for key, value := range a { + if other, found := b[key]; found { + value = value + other + } + result[key] = value + } + + for key, value := range b { + if _, found := result[key]; !found { + result[key] = value + } + } + return result +} + +// Subtract returns the result of a - b for each named resource +func Subtract(a ResourceList, b ResourceList) ResourceList { + result := ResourceList{} + for key, value := range a { + if other, found := b[key]; found { + value = value - other + } + result[key] = value + } + + for key, value := range b { + if _, found := result[key]; !found { + result[key] = -value + } + } + + return result +} + +// Zero returns the result of a - a for each named resource +func Zero(a ResourceList) ResourceList { + result := ResourceList{} + for key := range a { + result[key] = 0 + } + return result +} diff --git a/src/pkg/types/resources_test.go b/src/pkg/types/resources_test.go new file mode 100644 index 000000000..20f164707 --- /dev/null +++ b/src/pkg/types/resources_test.go @@ -0,0 +1,81 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type ResourcesSuite struct { + suite.Suite +} + +func (suite *ResourcesSuite) TestNewResourceList() { + res1, err1 := NewResourceList("") + suite.Error(err1) + suite.Nil(res1) + suite.Equal(0, len(res1)) + + res2, err2 := NewResourceList("{}") + suite.Nil(err2) + suite.NotNil(res2) +} + +func (suite *ResourcesSuite) TestEquals() { + suite.True(Equals(ResourceList{}, ResourceList{})) + suite.True(Equals(ResourceList{ResourceStorage: 100}, ResourceList{ResourceStorage: 100})) + suite.False(Equals(ResourceList{ResourceStorage: 100}, ResourceList{ResourceStorage: 200})) + suite.False(Equals(ResourceList{ResourceStorage: 100}, ResourceList{ResourceStorage: 100, ResourceCount: 10})) + suite.False(Equals(ResourceList{ResourceStorage: 100, ResourceCount: 10}, ResourceList{ResourceStorage: 100})) +} + +func (suite *ResourcesSuite) TestAdd() { + res1 := ResourceList{ResourceStorage: 100} + res2 := ResourceList{ResourceStorage: 100} + res3 := ResourceList{ResourceStorage: 100, ResourceCount: 10} + res4 := ResourceList{ResourceCount: 10} + + suite.Equal(res1, Add(ResourceList{}, res1)) + suite.Equal(ResourceList{ResourceStorage: 200}, Add(res1, res2)) + suite.Equal(ResourceList{ResourceStorage: 200, ResourceCount: 10}, Add(res1, res3)) + suite.Equal(ResourceList{ResourceStorage: 100, ResourceCount: 10}, Add(res1, res4)) +} + +func (suite *ResourcesSuite) TestSubtract() { + res1 := ResourceList{ResourceStorage: 100} + res2 := ResourceList{ResourceStorage: 100} + res3 := ResourceList{ResourceStorage: 100, ResourceCount: 10} + res4 := ResourceList{ResourceCount: 10} + + suite.Equal(res1, Subtract(res1, ResourceList{})) + suite.Equal(ResourceList{ResourceStorage: 0}, Subtract(res1, res2)) + suite.Equal(ResourceList{ResourceStorage: 0, ResourceCount: -10}, Subtract(res1, res3)) + suite.Equal(ResourceList{ResourceStorage: 100, ResourceCount: -10}, Subtract(res1, res4)) +} + +func (suite *ResourcesSuite) TestZero() { + res1 := ResourceList{ResourceStorage: 100} + res2 := ResourceList{ResourceCount: 10, ResourceStorage: 100} + + suite.Equal(ResourceList{}, Zero(ResourceList{})) + suite.Equal(ResourceList{ResourceStorage: 0}, Zero(res1)) + suite.Equal(ResourceList{ResourceStorage: 0, ResourceCount: 0}, Zero(res2)) +} + +func TestRunResourcesSuite(t *testing.T) { + suite.Run(t, new(ResourcesSuite)) +} diff --git a/src/portal/lib/src/config/config.ts b/src/portal/lib/src/config/config.ts index e24aa30dc..23078e106 100644 --- a/src/portal/lib/src/config/config.ts +++ b/src/portal/lib/src/config/config.ts @@ -97,6 +97,8 @@ export class Configuration { oidc_client_secret?: StringValueItem; oidc_verify_cert?: BoolValueItem; oidc_scope?: StringValueItem; + count_per_project: NumberValueItem; + storage_per_project: NumberValueItem; public constructor() { this.auth_mode = new StringValueItem("db_auth", true); this.project_creation_restriction = new StringValueItem("everyone", true); @@ -148,5 +150,7 @@ export class Configuration { this.oidc_client_secret = new StringValueItem('', true); this.oidc_verify_cert = new BoolValueItem(false, true); this.oidc_scope = new StringValueItem('', true); + this.count_per_project = new NumberValueItem(-1, true); + this.storage_per_project = new NumberValueItem(-1, true); } } diff --git a/src/portal/lib/src/config/index.ts b/src/portal/lib/src/config/index.ts index 5ecae2c6e..a43adbce6 100644 --- a/src/portal/lib/src/config/index.ts +++ b/src/portal/lib/src/config/index.ts @@ -6,6 +6,8 @@ import { VulnerabilityConfigComponent } from './vulnerability/vulnerability-conf import { RegistryConfigComponent } from './registry-config.component'; import { GcComponent } from './gc/gc.component'; import { GcHistoryComponent } from './gc/gc-history/gc-history.component'; +import { ProjectQuotasComponent } from './project-quotas/project-quotas.component'; +import { EditProjectQuotasComponent } from './project-quotas/edit-project-quotas/edit-project-quotas.component'; export * from './config'; export * from './replication/replication-config.component'; @@ -20,5 +22,7 @@ export const CONFIGURATION_DIRECTIVES: Type[] = [ GcComponent, SystemSettingsComponent, VulnerabilityConfigComponent, - RegistryConfigComponent + RegistryConfigComponent, + ProjectQuotasComponent, + EditProjectQuotasComponent ]; diff --git a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html new file mode 100644 index 000000000..98e57589a --- /dev/null +++ b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html @@ -0,0 +1,86 @@ + +

{{ defaultTextsObj.editQuota }}

+ +
+ + +
+
+ + +
+ + + {{'PROJECT.QUOTA_UNLIMIT_TIP' | translate }} + +
+
+ +
+ +
+
+
+ + +
+ +
+ + + {{'PROJECT.QUOTA_UNLIMIT_TIP' | translate }} + +
+
+ +
+ +
+
+
+
+
+ + +
+ \ No newline at end of file diff --git a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.scss b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.scss new file mode 100644 index 000000000..024a058d6 --- /dev/null +++ b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.scss @@ -0,0 +1,61 @@ + +::ng-deep .modal-dialog { + width: 25rem; +} +.modal-body { + padding-top: 0.8rem; + overflow-y: visible; + overflow-x: visible; + .clr-form-compact { + div.form-group { + padding-left: 8.5rem; + .mr-3px { + margin-right: 3px; + } + .quota-input { + width: 2rem; + padding-right: 0.8rem; + } + .select-div { + width: 2.5rem; + + ::ng-deep .clr-form-control { + margin-top: 0.28rem; + + select { + padding-right: 15px; + } + } + } + } + } + + .clr-form-compact-common { + div.form-group { + padding-left: 6rem; + + .select-div { + width: 1.6rem; + } + } + } +} + +.progress-block { + width: 8rem; +} + +.progress-div { + position: relative; + padding-right: 0.6rem; + width: 9rem; +} + +.progress-label { + position: absolute; + right: -2.3rem; + top: 0; + width: 3.5rem; + font-weight: 100; + font-size: 10px; +} \ No newline at end of file diff --git a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.spec.ts b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.spec.ts new file mode 100644 index 000000000..595f1ab1b --- /dev/null +++ b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.spec.ts @@ -0,0 +1,37 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { EditProjectQuotasComponent } from './edit-project-quotas.component'; +import { SharedModule } from '../../../shared/shared.module'; +import { InlineAlertComponent } from '../../../inline-alert/inline-alert.component'; +import { SERVICE_CONFIG, IServiceConfig } from '../../../service.config'; +import { RouterModule } from '@angular/router'; + +describe('EditProjectQuotasComponent', () => { + let component: EditProjectQuotasComponent; + let fixture: ComponentFixture; + let config: IServiceConfig = { + quotaUrl: "/api/quotas/testing" + }; + beforeEach(async(() => { + TestBed.configureTestingModule({ + imports: [ + SharedModule, + RouterModule.forRoot([]) + ], + declarations: [ EditProjectQuotasComponent, InlineAlertComponent ], + providers: [ + { provide: SERVICE_CONFIG, useValue: config }, + ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(EditProjectQuotasComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.ts b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.ts new file mode 100644 index 000000000..a0e435c91 --- /dev/null +++ b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.ts @@ -0,0 +1,143 @@ +import { + Component, + EventEmitter, + Output, + ViewChild, + OnInit, +} from '@angular/core'; +import { NgForm, Validators } from '@angular/forms'; +import { ActivatedRoute } from "@angular/router"; + +import { TranslateService } from '@ngx-translate/core'; + +import { InlineAlertComponent } from '../../../inline-alert/inline-alert.component'; + +import { QuotaUnits, QuotaUnlimited } from "../../../shared/shared.const"; + +import { clone, getSuitableUnit, getByte, GetIntegerAndUnit, validateLimit } from '../../../utils'; +import { EditQuotaQuotaInterface, QuotaHardLimitInterface } from '../../../service'; +import { distinctUntilChanged } from 'rxjs/operators'; + +@Component({ + selector: 'edit-project-quotas', + templateUrl: './edit-project-quotas.component.html', + styleUrls: ['./edit-project-quotas.component.scss'] +}) +export class EditProjectQuotasComponent implements OnInit { + openEditQuota: boolean; + defaultTextsObj: { editQuota: string; setQuota: string; countQuota: string; storageQuota: string; isSystemDefaultQuota: boolean } = { + editQuota: '', + setQuota: '', + countQuota: '', + storageQuota: '', + isSystemDefaultQuota: false, + }; + quotaHardLimitValue: QuotaHardLimitInterface = { + storageLimit: -1 + , storageUnit: '' + , countLimit: -1 + }; + quotaUnits = QuotaUnits; + staticBackdrop = true; + closable = false; + quotaForm: NgForm; + @ViewChild(InlineAlertComponent) + inlineAlert: InlineAlertComponent; + + @ViewChild('quotaForm') + currentForm: NgForm; + @Output() confirmAction = new EventEmitter(); + constructor( + private translateService: TranslateService, + private route: ActivatedRoute) { } + + ngOnInit() { + } + + onSubmit(): void { + const emitData = { + formValue: this.currentForm.value, + isSystemDefaultQuota: this.defaultTextsObj.isSystemDefaultQuota, + id: this.quotaHardLimitValue.id + }; + this.confirmAction.emit(emitData); + } + onCancel() { + this.openEditQuota = false; + } + + openEditQuotaModal(defaultTextsObj: EditQuotaQuotaInterface): void { + this.defaultTextsObj = defaultTextsObj; + if (this.defaultTextsObj.isSystemDefaultQuota) { + this.quotaHardLimitValue = { + storageLimit: defaultTextsObj.quotaHardLimitValue.storageLimit === QuotaUnlimited ? + QuotaUnlimited : GetIntegerAndUnit(defaultTextsObj.quotaHardLimitValue.storageLimit + , clone(QuotaUnits), 0, clone(QuotaUnits)).partNumberHard + , storageUnit: defaultTextsObj.quotaHardLimitValue.storageLimit === QuotaUnlimited ? + QuotaUnits[3].UNIT : GetIntegerAndUnit(defaultTextsObj.quotaHardLimitValue.storageLimit + , clone(QuotaUnits), 0, clone(QuotaUnits)).partCharacterHard + , countLimit: defaultTextsObj.quotaHardLimitValue.countLimit + }; + } else { + this.quotaHardLimitValue = { + storageLimit: defaultTextsObj.quotaHardLimitValue.hard.storage === QuotaUnlimited ? + QuotaUnlimited : GetIntegerAndUnit(defaultTextsObj.quotaHardLimitValue.hard.storage + , clone(QuotaUnits), defaultTextsObj.quotaHardLimitValue.used.storage, clone(QuotaUnits)).partNumberHard + , storageUnit: defaultTextsObj.quotaHardLimitValue.hard.storage === QuotaUnlimited ? + QuotaUnits[3].UNIT : GetIntegerAndUnit(defaultTextsObj.quotaHardLimitValue.hard.storage + , clone(QuotaUnits), defaultTextsObj.quotaHardLimitValue.used.storage, clone(QuotaUnits)).partCharacterHard + , countLimit: defaultTextsObj.quotaHardLimitValue.hard.count + , id: defaultTextsObj.quotaHardLimitValue.id + , countUsed: defaultTextsObj.quotaHardLimitValue.used.count + , storageUsed: defaultTextsObj.quotaHardLimitValue.used.storage + }; + } + let defaultForm = { + count: this.quotaHardLimitValue.countLimit + , storage: this.quotaHardLimitValue.storageLimit + , storageUnit: this.quotaHardLimitValue.storageUnit + }; + this.currentForm.resetForm(defaultForm); + this.openEditQuota = true; + + this.currentForm.form.controls['storage'].setValidators( + [ + Validators.required, + Validators.pattern('(^-1$)|(^([1-9]+)([0-9]+)*$)'), + validateLimit(this.currentForm.form.controls['storageUnit']) + ]); + this.currentForm.form.valueChanges + .pipe(distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b))) + .subscribe((data) => { + ['storage', 'storageUnit'].forEach(fieldName => { + if (this.currentForm.form.get(fieldName) && this.currentForm.form.get(fieldName).value !== null) { + this.currentForm.form.get(fieldName).updateValueAndValidity(); + } + }); + }); + } + + get isValid() { + return this.currentForm.valid && this.currentForm.dirty; + } + getSuitableUnit(value) { + const QuotaUnitsCopy = clone(QuotaUnits); + return getSuitableUnit(value, QuotaUnitsCopy); + } + getIntegerAndUnit(valueHard, valueUsed) { + return GetIntegerAndUnit(valueHard + , clone(QuotaUnits), valueUsed, clone(QuotaUnits)); + } + getByte(count: number, unit: string) { + if (+count === +count) { + return getByte(+count, unit); + } + return 0; + } + getDangerStyle(limit: number | string, used: number | string, unit?: string) { + if (unit) { + return limit !== QuotaUnlimited ? +used / getByte(+limit, unit) > 0.9 : false; + } + return limit !== QuotaUnlimited ? +used / +limit > 0.9 : false; + } +} diff --git a/src/portal/lib/src/config/project-quotas/project-quotas.component.html b/src/portal/lib/src/config/project-quotas/project-quotas.component.html new file mode 100644 index 000000000..6f9e9cad2 --- /dev/null +++ b/src/portal/lib/src/config/project-quotas/project-quotas.component.html @@ -0,0 +1,75 @@ +
+
+
+
+
+
{{'QUOTA.PROJECT_QUOTA_DEFAULT_ARTIFACT' | translate}}{{ quotaHardLimitValue?.countLimit === -1? ('QUOTA.UNLIMITED'| translate): quotaHardLimitValue?.countLimit }} +
+
{{'QUOTA.PROJECT_QUOTA_DEFAULT_DISK' | translate}} + {{ quotaHardLimitValue?.storageLimit === -1?('QUOTA.UNLIMITED' | translate): getIntegerAndUnit(quotaHardLimitValue?.storageLimit, 0).partNumberHard}} + {{ quotaHardLimitValue?.storageLimit === -1?'':quotaHardLimitValue?.storageUnit }} +
+
+ +
+
+ + + +
+
+
+ + {{'QUOTA.PROJECT' | translate}} + {{'QUOTA.OWNER' | translate}} + {{'QUOTA.COUNT' | translate }} + {{'QUOTA.STORAGE' | translate }} + {{'QUOTA.PLACEHOLDER' | translate }} + + + + + + {{quota?.ref?.name}} + {{quota?.ref?.owner_name}} + +
+
+ +
+ +
+ + +
+
+ +
+ +
+ + + + {{pagination.firstItem + 1}} - {{pagination.lastItem + 1}} + {{'DESTINATION.OF' | translate}} + {{totalCount}} {{'SUMMARY.QUOTAS' | translate}} + + + +
+
+ +
\ No newline at end of file diff --git a/src/portal/lib/src/config/project-quotas/project-quotas.component.scss b/src/portal/lib/src/config/project-quotas/project-quotas.component.scss new file mode 100644 index 000000000..eeb09db91 --- /dev/null +++ b/src/portal/lib/src/config/project-quotas/project-quotas.component.scss @@ -0,0 +1,42 @@ +.default-quota { + display: flex; + + .default-quota-text { + display: flex; + justify-content: space-between; + min-width: 13rem; + + .num-count { + display: inline-block; + min-width: 2rem; + } + } +} + +.color-0 { + color: #000; +} + +.progress-block { + label { + font-weight: 400 !important; + } +} + +.default-quota-edit-button { + height: 1rem; +} + +.min-label-width { + min-width: 120px; +} + +.quota-top { + display: flex; + justify-content: space-between; +} + +.refresh-div { + margin-top: auto; + cursor: pointer; +} \ No newline at end of file diff --git a/src/portal/lib/src/config/project-quotas/project-quotas.component.spec.ts b/src/portal/lib/src/config/project-quotas/project-quotas.component.spec.ts new file mode 100644 index 000000000..168685550 --- /dev/null +++ b/src/portal/lib/src/config/project-quotas/project-quotas.component.spec.ts @@ -0,0 +1,93 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { ProjectQuotasComponent } from './project-quotas.component'; +import { IServiceConfig, SERVICE_CONFIG } from '../../service.config'; +import { SharedModule } from '../../shared/shared.module'; +import { RouterModule } from '@angular/router'; +import { EditProjectQuotasComponent } from './edit-project-quotas/edit-project-quotas.component'; +import { InlineAlertComponent } from '../../inline-alert/inline-alert.component'; +import { + ConfigurationService, ConfigurationDefaultService, QuotaService + , QuotaDefaultService, Quota, RequestQueryParams +} from '../../service'; +import { ErrorHandler } from '../../error-handler'; +import { of } from 'rxjs'; +import { delay } from 'rxjs/operators'; +import {APP_BASE_HREF} from '@angular/common'; +describe('ProjectQuotasComponent', () => { + let spy: jasmine.Spy; + let quotaService: QuotaService; + + let component: ProjectQuotasComponent; + let fixture: ComponentFixture; + + let config: IServiceConfig = { + quotaUrl: "/api/quotas/testing" + }; + let mockQuotaList: Quota[] = [{ + id: 1111, + ref: { + id: 1111, + name: "project1", + owner_name: "project1" + }, + creation_time: "12212112121", + update_time: "12212112121", + hard: { + count: -1, + storage: -1, + }, + used: { + count: 1234, + storage: 1234 + }, + } + ]; + beforeEach(async(() => { + TestBed.configureTestingModule({ + imports: [ + SharedModule, + RouterModule.forRoot([]) + ], + declarations: [ProjectQuotasComponent, EditProjectQuotasComponent, InlineAlertComponent], + providers: [ + ErrorHandler, + { provide: SERVICE_CONFIG, useValue: config }, + { provide: ConfigurationService, useClass: ConfigurationDefaultService }, + { provide: QuotaService, useClass: QuotaDefaultService }, + { provide: APP_BASE_HREF, useValue : '/' } + + ] + }) + .compileComponents(); + })); + + beforeEach(async(() => { + + fixture = TestBed.createComponent(ProjectQuotasComponent); + component = fixture.componentInstance; + component.quotaHardLimitValue = { + countLimit: 1111, + storageLimit: 23, + storageUnit: 'GB' + }; + component.loading = true; + quotaService = fixture.debugElement.injector.get(QuotaService); + spy = spyOn(quotaService, 'getQuotaList') + .and.callFake(function (params: RequestQueryParams) { + let header = new Map(); + header.set("X-Total-Count", 123); + const httpRes = { + headers: header, + body: mockQuotaList + }; + return of(httpRes).pipe(delay(0)); + }); + + fixture.detectChanges(); + })); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/portal/lib/src/config/project-quotas/project-quotas.component.ts b/src/portal/lib/src/config/project-quotas/project-quotas.component.ts new file mode 100644 index 000000000..ffaff1336 --- /dev/null +++ b/src/portal/lib/src/config/project-quotas/project-quotas.component.ts @@ -0,0 +1,238 @@ +import { Component, Input, Output, EventEmitter, ViewChild, SimpleChanges, OnChanges } from '@angular/core'; +import { Configuration } from '../config'; +import { + Quota, State, Comparator, ClrDatagridComparatorInterface, QuotaHardLimitInterface, QuotaHard +} from '../../service/interface'; +import { + clone, isEmpty, getChanges, getSuitableUnit, calculatePage, CustomComparator + , getByte, GetIntegerAndUnit +} from '../../utils'; +import { ErrorHandler } from '../../error-handler/index'; +import { QuotaUnits, QuotaUnlimited } from '../../shared/shared.const'; +import { EditProjectQuotasComponent } from './edit-project-quotas/edit-project-quotas.component'; +import { + ConfigurationService +} from '../../service/index'; +import { TranslateService } from '@ngx-translate/core'; +import { forkJoin } from 'rxjs'; +import { QuotaService } from "../../service/quota.service"; +import { Router } from '@angular/router'; +import { finalize } from 'rxjs/operators'; +const quotaSort = { + count: 'used.count', + storage: "used.storage", + sortType: 'string' +}; +const QuotaType = 'project'; + +@Component({ + selector: 'project-quotas', + templateUrl: './project-quotas.component.html', + styleUrls: ['./project-quotas.component.scss'] +}) +export class ProjectQuotasComponent implements OnChanges { + + config: Configuration = new Configuration(); + @ViewChild('editProjectQuotas') + editQuotaDialog: EditProjectQuotasComponent; + loading = true; + quotaHardLimitValue: QuotaHardLimitInterface; + currentState: State; + + @Output() configChange: EventEmitter = new EventEmitter(); + @Output() refreshAllconfig: EventEmitter = new EventEmitter(); + quotaList: Quota[] = []; + originalConfig: Configuration; + currentPage = 1; + totalCount = 0; + pageSize = 15; + @Input() + get allConfig(): Configuration { + return this.config; + } + set allConfig(cfg: Configuration) { + this.config = cfg; + this.configChange.emit(this.config); + } + countComparator: Comparator = new CustomComparator(quotaSort.count, quotaSort.sortType); + storageComparator: Comparator = new CustomComparator(quotaSort.storage, quotaSort.sortType); + + constructor( + private configService: ConfigurationService, + private quotaService: QuotaService, + private translate: TranslateService, + private router: Router, + private errorHandler: ErrorHandler) { } + + editQuota(quotaHardLimitValue: QuotaHardLimitInterface) { + const defaultTexts = [this.translate.get('QUOTA.EDIT_PROJECT_QUOTAS'), this.translate.get('QUOTA.SET_QUOTAS') + , this.translate.get('QUOTA.COUNT_QUOTA'), this.translate.get('QUOTA.STORAGE_QUOTA')]; + forkJoin(...defaultTexts).subscribe(res => { + const defaultTextsObj = { + editQuota: res[0], + setQuota: res[1], + countQuota: res[2], + storageQuota: res[3], + quotaHardLimitValue: quotaHardLimitValue, + isSystemDefaultQuota: false + }; + this.editQuotaDialog.openEditQuotaModal(defaultTextsObj); + }); + } + + editDefaultQuota(quotaHardLimitValue: QuotaHardLimitInterface) { + const defaultTexts = [this.translate.get('QUOTA.EDIT_DEFAULT_PROJECT_QUOTAS'), this.translate.get('QUOTA.SET_DEFAULT_QUOTAS') + , this.translate.get('QUOTA.COUNT_DEFAULT_QUOTA'), this.translate.get('QUOTA.STORAGE_DEFAULT_QUOTA')]; + forkJoin(...defaultTexts).subscribe(res => { + const defaultTextsObj = { + editQuota: res[0], + setQuota: res[1], + countQuota: res[2], + storageQuota: res[3], + quotaHardLimitValue: quotaHardLimitValue, + isSystemDefaultQuota: true + }; + this.editQuotaDialog.openEditQuotaModal(defaultTextsObj); + + }); + } + public getChanges() { + let allChanges = getChanges(this.originalConfig, this.config); + if (allChanges) { + return this.getQuotaChanges(allChanges); + } + return null; + } + + getQuotaChanges(allChanges) { + let changes = {}; + for (let prop in allChanges) { + if (prop === 'storage_per_project' + || prop === 'count_per_project' + ) { + changes[prop] = allChanges[prop]; + } + } + return changes; + } + + public saveConfig(configQuota): void { + this.allConfig.count_per_project.value = configQuota.count; + this.allConfig.storage_per_project.value = +configQuota.storage === QuotaUnlimited ? + configQuota.storage : getByte(configQuota.storage, configQuota.storageUnit); + let changes = this.getChanges(); + if (!isEmpty(changes)) { + this.loading = true; + this.configService.saveConfigurations(changes) + .pipe(finalize(() => { + this.loading = false; + this.editQuotaDialog.openEditQuota = false; + })) + .subscribe(response => { + this.refreshAllconfig.emit(); + this.errorHandler.info('CONFIG.SAVE_SUCCESS'); + } + , error => { + this.errorHandler.error(error); + }); + } else { + // Inprop situation, should not come here + this.translate.get('CONFIG.NO_CHANGE').subscribe(res => { + this.editQuotaDialog.inlineAlert.showInlineError(res); + }); + } + } + + confirmEdit(event) { + if (event.isSystemDefaultQuota) { + this.saveConfig(event.formValue); + } else { + this.saveCurrentQuota(event); + } + } + saveCurrentQuota(event) { + let count = +event.formValue.count; + let storage = +event.formValue.storage === QuotaUnlimited ? + +event.formValue.storage : getByte(+event.formValue.storage, event.formValue.storageUnit); + let rep: QuotaHard = { hard: { count, storage } }; + this.loading = true; + this.quotaService.updateQuota(event.id, rep).subscribe(res => { + this.editQuotaDialog.openEditQuota = false; + this.getQuotaList(this.currentState); + this.errorHandler.info('QUOTA.SAVE_SUCCESS'); + }, error => { + this.errorHandler.error(error); + this.loading = false; + }); + } + + getquotaHardLimitValue() { + const storageNumberAndUnit = this.allConfig.storage_per_project ? this.allConfig.storage_per_project.value : QuotaUnlimited; + const storageLimit = storageNumberAndUnit; + const storageUnit = this.getIntegerAndUnit(storageNumberAndUnit, 0).partCharacterHard; + const countLimit = this.allConfig.count_per_project ? this.allConfig.count_per_project.value : QuotaUnlimited; + this.quotaHardLimitValue = { storageLimit, storageUnit, countLimit }; + } + getQuotaList(state: State) { + if (!state || !state.page) { + return; + } + // Keep state for future filtering and sorting + this.currentState = state; + + let pageNumber: number = calculatePage(state); + if (pageNumber <= 0) { pageNumber = 1; } + let sortBy: any = ''; + if (state.sort) { + sortBy = state.sort.by as string | ClrDatagridComparatorInterface; + sortBy = sortBy.fieldName ? sortBy.fieldName : sortBy; + sortBy = state.sort.reverse ? `-${sortBy}` : sortBy; + } + this.loading = true; + + this.quotaService.getQuotaList(QuotaType, pageNumber, this.pageSize, sortBy).pipe(finalize(() => { + this.loading = false; + })).subscribe(res => { + if (res.headers) { + let xHeader: string = res.headers.get("X-Total-Count"); + if (xHeader) { + this.totalCount = parseInt(xHeader, 0); + } + } + this.quotaList = res.body.filter((quota) => { + return quota.ref !== null; + }) as Quota[]; + }, error => { + this.errorHandler.error(error); + }); + } + ngOnChanges(changes: SimpleChanges): void { + if (changes && changes["allConfig"]) { + this.originalConfig = clone(this.config); + this.getquotaHardLimitValue(); + } + } + getSuitableUnit(value) { + const QuotaUnitsCopy = clone(QuotaUnits); + return getSuitableUnit(value, QuotaUnitsCopy); + } + getIntegerAndUnit(valueHard, valueUsed) { + return GetIntegerAndUnit(valueHard + , clone(QuotaUnits), valueUsed, clone(QuotaUnits)); + } + + goToLink(proId) { + let linkUrl = ["harbor", "projects", proId, "summary"]; + this.router.navigate(linkUrl); + } + refresh() { + const state: State = { + page: { + from: 0, + to: 14, + size: 15 + }, + }; + this.getQuotaList(state); + } +} diff --git a/src/portal/lib/src/harbor-library.module.ts b/src/portal/lib/src/harbor-library.module.ts index 4c764b439..51babecf6 100644 --- a/src/portal/lib/src/harbor-library.module.ts +++ b/src/portal/lib/src/harbor-library.module.ts @@ -38,6 +38,8 @@ import { EndpointDefaultService, ReplicationService, ReplicationDefaultService, + QuotaService, + QuotaDefaultService, RepositoryService, RepositoryDefaultService, TagService, @@ -131,6 +133,9 @@ export interface HarborModuleConfig { // Service implementation for replication replicationService?: Provider; + // Service implementation for replication + QuotaService?: Provider; + // Service implementation for repository repositoryService?: Provider; @@ -257,6 +262,7 @@ export class HarborLibraryModule { config.logService || { provide: AccessLogService, useClass: AccessLogDefaultService }, config.endpointService || { provide: EndpointService, useClass: EndpointDefaultService }, config.replicationService || { provide: ReplicationService, useClass: ReplicationDefaultService }, + config.QuotaService || { provide: QuotaService, useClass: QuotaDefaultService }, config.repositoryService || { provide: RepositoryService, useClass: RepositoryDefaultService }, config.tagService || { provide: TagService, useClass: TagDefaultService }, config.retagService || { provide: RetagService, useClass: RetagDefaultService }, @@ -295,6 +301,7 @@ export class HarborLibraryModule { config.logService || { provide: AccessLogService, useClass: AccessLogDefaultService }, config.endpointService || { provide: EndpointService, useClass: EndpointDefaultService }, config.replicationService || { provide: ReplicationService, useClass: ReplicationDefaultService }, + config.QuotaService || { provide: QuotaService, useClass: QuotaDefaultService }, config.repositoryService || { provide: RepositoryService, useClass: RepositoryDefaultService }, config.tagService || { provide: TagService, useClass: TagDefaultService }, config.retagService || { provide: RetagService, useClass: RetagDefaultService }, diff --git a/src/portal/lib/src/service.config.ts b/src/portal/lib/src/service.config.ts index b5bca0cd4..545e60e0a 100644 --- a/src/portal/lib/src/service.config.ts +++ b/src/portal/lib/src/service.config.ts @@ -232,4 +232,6 @@ export interface IServiceConfig { gcEndpoint?: string; ScanAllEndpoint?: string; + + quotaUrl?: string; } diff --git a/src/portal/lib/src/service/index.ts b/src/portal/lib/src/service/index.ts index 2c22c3bd8..311501a80 100644 --- a/src/portal/lib/src/service/index.ts +++ b/src/portal/lib/src/service/index.ts @@ -14,3 +14,4 @@ export * from "./label.service"; export * from "./retag.service"; export * from "./permission.service"; export * from "./permission-static"; +export * from "./quota.service"; diff --git a/src/portal/lib/src/service/interface.ts b/src/portal/lib/src/service/interface.ts index f1410c056..5de814f2b 100644 --- a/src/portal/lib/src/service/interface.ts +++ b/src/portal/lib/src/service/interface.ts @@ -99,9 +99,9 @@ export interface PingEndpoint extends Base { } export interface Filter { - type: string; - style: string; - values ?: string[]; + type: string; + style: string; + values?: string[]; } /** @@ -122,7 +122,7 @@ export interface ReplicationRule extends Base { deletion?: boolean; src_registry?: any; dest_registry?: any; - src_namespaces: string []; + src_namespaces: string[]; dest_namespace?: string; enabled: boolean; override: boolean; @@ -333,6 +333,33 @@ export interface Label { scope: string; project_id: number; } + +export interface Quota { + id: number; + ref: { + name: string; + owner_name: string; + id: number; + } | null; + creation_time: string; + update_time: string; + hard: { + count: number; + storage: number; + }; + used: { + count: number; + storage: number; + }; +} +export interface QuotaHard { + hard: QuotaCountStorage; +} +export interface QuotaCountStorage { + count: number; + storage: number; +} + export interface CardItemEvent { event_type: string; item: any; @@ -408,26 +435,26 @@ export class OriginCron { cron: string; } -export interface HttpOptionInterface { +export interface HttpOptionInterface { headers?: HttpHeaders | { - [header: string]: string | string[]; + [header: string]: string | string[]; }; observe?: 'body'; params?: HttpParams | { - [param: string]: string | string[]; + [param: string]: string | string[]; }; reportProgress?: boolean; responseType: 'json'; withCredentials?: boolean; } -export interface HttpOptionTextInterface { +export interface HttpOptionTextInterface { headers?: HttpHeaders | { - [header: string]: string | string[]; + [header: string]: string | string[]; }; observe?: 'body'; params?: HttpParams | { - [param: string]: string | string[]; + [param: string]: string | string[]; }; reportProgress?: boolean; responseType: 'text'; @@ -435,14 +462,38 @@ export interface HttpOptionTextInterface { } -export interface ProjectRootInterface { +export interface ProjectRootInterface { NAME: string; VALUE: number; LABEL: string; } export interface SystemCVEWhitelist { - id: number; - project_id: number; - expires_at: number; - items: Array<{ "cve_id": string; }>; + id: number; + project_id: number; + expires_at: number; + items: Array<{ "cve_id": string; }>; +} +export interface QuotaHardInterface { + count_per_project: number; + storage_per_project: number; +} + +export interface QuotaUnitInterface { + UNIT: string; +} +export interface QuotaHardLimitInterface { + countLimit: number; + storageLimit: number; + storageUnit: string; + id?: string; + countUsed?: string; + storageUsed?: string; +} +export interface EditQuotaQuotaInterface { + editQuota: string; + setQuota: string; + countQuota: string; + storageQuota: string; + quotaHardLimitValue: QuotaHardLimitInterface | any; + isSystemDefaultQuota: boolean; } diff --git a/src/portal/lib/src/service/permission-static.ts b/src/portal/lib/src/service/permission-static.ts index da0aaf62c..b0ef1fa25 100644 --- a/src/portal/lib/src/service/permission-static.ts +++ b/src/portal/lib/src/service/permission-static.ts @@ -1,8 +1,10 @@ export const USERSTATICPERMISSION = { "PROJECT": { - 'KEY': 'project', + 'KEY': '.', 'VALUE': { - "DELETE": "delete" + "DELETE": "delete", + "UPDATE": "update", + "READ": "read", } }, "MEMBER": { diff --git a/src/portal/lib/src/service/project.service.ts b/src/portal/lib/src/service/project.service.ts index e6f6ef0a8..a0f34ad4c 100644 --- a/src/portal/lib/src/service/project.service.ts +++ b/src/portal/lib/src/service/project.service.ts @@ -69,11 +69,12 @@ export abstract class ProjectService { page?: number, pageSize?: number ): Observable>; - abstract createProject(name: string, metadata: any): Observable; + abstract createProject(name: string, metadata: any, countLimit: number, storageLimit: number): Observable; abstract toggleProjectPublic(projectId: number, isPublic: string): Observable; abstract deleteProject(projectId: number): Observable; abstract checkProjectExists(projectName: string): Observable; abstract checkProjectMember(projectId: number): Observable; + abstract getProjectSummary(projectId: number): Observable; } /** @@ -149,12 +150,14 @@ export class ProjectDefaultService extends ProjectService { catchError(error => observableThrowError(error)), ); } - public createProject(name: string, metadata: any): Observable { + public createProject(name: string, metadata: any, countLimit: number, storageLimit: number): Observable { return this.http .post(`/api/projects`, JSON.stringify({'project_name': name, 'metadata': { public: metadata.public ? 'true' : 'false', - }}) + }, + count_limit: countLimit, storage_limit: storageLimit + }) , HTTP_JSON_OPTIONS).pipe( catchError(error => observableThrowError(error)), ); } @@ -182,4 +185,9 @@ export class ProjectDefaultService extends ProjectService { .get(`/api/projects/${projectId}/members`, HTTP_GET_OPTIONS).pipe( catchError(error => observableThrowError(error)), ); } + public getProjectSummary(projectId: number): Observable { + return this.http + .get(`/api/projects/${projectId}/summary`, HTTP_GET_OPTIONS).pipe( + catchError(error => observableThrowError(error)), ); + } } diff --git a/src/portal/lib/src/service/quota.service.ts b/src/portal/lib/src/service/quota.service.ts new file mode 100644 index 000000000..a5f7a0853 --- /dev/null +++ b/src/portal/lib/src/service/quota.service.ts @@ -0,0 +1,92 @@ +import { HttpClient, HttpResponse, HttpParams } from "@angular/common/http"; +import { Injectable, Inject } from "@angular/core"; +import { + HTTP_JSON_OPTIONS, + buildHttpRequestOptionsWithObserveResponse, +} from "../utils"; +import { + QuotaHard +} from "./interface"; +import { map, catchError } from "rxjs/operators"; +import { Observable, throwError as observableThrowError } from "rxjs"; +import { Quota } from "./interface"; +import { SERVICE_CONFIG, IServiceConfig } from "../service.config"; +/** + * Define the service methods to handle the replication (rule and job) related things. + * + ** + * @abstract + * class QuotaService + */ +export abstract class QuotaService { + /** + * + * @abstract + * returns {(Observable)} + * + * @memberOf QuotaService + */ + abstract getQuotaList(quotaType, page?, pageSize?, sortBy?: any): + any; + + abstract updateQuota( + id: number, + rep: QuotaHard + ): Observable; +} + +/** + * Implement default service for replication rule and job. + * + ** + * class QuotaDefaultService + * extends {QuotaService} + */ +@Injectable() +export class QuotaDefaultService extends QuotaService { + quotaUrl: string; + constructor( + private http: HttpClient, + @Inject(SERVICE_CONFIG) private config: IServiceConfig + ) { + super(); + if (this.config && this.config.quotaUrl) { + this.quotaUrl = this.config.quotaUrl; + } + } + + public getQuotaList(quotaType: string, page?, pageSize?, sortBy?: any): + any { + + let params = new HttpParams(); + if (quotaType) { + params = params.set('reference', quotaType); + } + if (page && pageSize) { + params = params.set('page', page + '').set('page_size', pageSize + ''); + } + if (sortBy) { + params = params.set('sort', sortBy); + } + + return this.http + .get>(this.quotaUrl + , buildHttpRequestOptionsWithObserveResponse(params)) + .pipe(map(response => { + return response; + }) + , catchError(error => observableThrowError(error))); + } + + public updateQuota( + id: number, + quotaHardLimit: QuotaHard + ): Observable { + + let url = `${this.quotaUrl}/${id}`; + return this.http + .put(url, quotaHardLimit, HTTP_JSON_OPTIONS) + .pipe(catchError(error => observableThrowError(error))); + } + +} diff --git a/src/portal/lib/src/shared/shared.const.ts b/src/portal/lib/src/shared/shared.const.ts index 84a22b8b7..db48e9354 100644 --- a/src/portal/lib/src/shared/shared.const.ts +++ b/src/portal/lib/src/shared/shared.const.ts @@ -71,6 +71,31 @@ export const FilterType = { export const enum ConfirmationButtons { CONFIRM_CANCEL, YES_NO, DELETE_CANCEL, CLOSE, REPLICATE_CANCEL, STOP_CANCEL } +export const QuotaUnits = [ + { + UNIT: "Byte", + }, + { + UNIT: "KB", + }, + { + UNIT: "MB", + }, + { + UNIT: "GB", + }, + { + UNIT: "TB", + }, +]; +export const QuotaUnlimited = -1; +export const StorageMultipleConstant = 1024; +export enum QuotaUnit { + TB = "TB", GB = "GB", MB = "MB", KB = "KB", BIT = "Byte" +} +export enum QuotaProgress { + COUNT_USED = "COUNT_USED", COUNT_HARD = "COUNT_HARD", STROAGE_USED = "STORAGE_USED", STORAGE_HARD = "STORAGE_HARD" +} export const LabelColor = [ { 'color': '#000000', 'textColor': 'white' }, { 'color': '#61717D', 'textColor': 'white' }, diff --git a/src/portal/lib/src/utils.ts b/src/portal/lib/src/utils.ts index a4500b8b5..7c3ede678 100644 --- a/src/portal/lib/src/utils.ts +++ b/src/portal/lib/src/utils.ts @@ -1,10 +1,11 @@ import { Observable } from "rxjs"; -import { HttpHeaders, HttpParams } from '@angular/common/http'; +import { HttpHeaders } from '@angular/common/http'; import { RequestQueryParams } from './service/RequestQueryParams'; import { DebugElement } from '@angular/core'; -import { Comparator, State, HttpOptionInterface, HttpOptionTextInterface } from './service/interface'; - +import { Comparator, State, HttpOptionInterface, HttpOptionTextInterface, QuotaUnitInterface } from './service/interface'; +import { QuotaUnits, StorageMultipleConstant } from './shared/shared.const'; +import { AbstractControl } from "@angular/forms"; /** * Convert the different async channels to the Promise type. * @@ -270,8 +271,8 @@ export function doFiltering(items: T[] if (filter['property'].indexOf('.') !== -1) { let arr = filter['property'].split('.'); if (Array.isArray(item[arr[0]]) && item[arr[0]].length) { - return item[arr[0]].some((data: any) => { - return filter['value'] === data[arr[1]]; + return item[arr[0]].some((data: any) => { + return filter['value'] === data[arr[1]]; }); } } else { @@ -382,14 +383,14 @@ export function isEmpty(obj: any): boolean { export function downloadFile(fileData) { let url = window.URL.createObjectURL(fileData.data); - let a = document.createElement("a"); - document.body.appendChild(a); - a.setAttribute("style", "display: none"); - a.href = url; - a.download = fileData.filename; - a.click(); - window.URL.revokeObjectURL(url); - a.remove(); + let a = document.createElement("a"); + document.body.appendChild(a); + a.setAttribute("style", "display: none"); + a.href = url; + a.download = fileData.filename; + a.click(); + window.URL.revokeObjectURL(url); + a.remove(); } export function getChanges(original: any, afterChange: any): { [key: string]: any | any[] } { @@ -429,3 +430,89 @@ export function cronRegex(testValue: any): boolean { let reg = new RegExp(regEx, "i"); return reg.test(testValue.trim()); } + +/** + * Keep decimal digits + * @param count number + * @param decimals number 1、2、3 ··· + */ +export const roundDecimals = (count, decimals = 0) => { + return Number(`${Math.round(+`${count}e${decimals}`)}e-${decimals}`); +}; +/** + * get suitable unit + * @param count number ;bit + * @param quotaUnitsDeep Array link QuotaUnits; + */ +export const getSuitableUnit = (count: number, quotaUnitsDeep: QuotaUnitInterface[]): string => { + for (let unitObj of quotaUnitsDeep) { + if (count / StorageMultipleConstant >= 1 && quotaUnitsDeep.length > 1) { + quotaUnitsDeep.shift(); + return getSuitableUnit(count / StorageMultipleConstant, quotaUnitsDeep); + } else { + return +count ? `${roundDecimals(count, 2)}${unitObj.UNIT}` : `0${unitObj.UNIT}`; + } + } + return `${roundDecimals(count, 2)}${QuotaUnits[0].UNIT}`; +}; +/** + * get byte from GB、MB、TB + * @param count number + * @param unit MB /GB / TB + */ +export const getByte = (count: number, unit: string): number => { + let flagIndex; + return QuotaUnits.reduce((totalValue, currentValue, index) => { + if (currentValue.UNIT === unit) { + flagIndex = index; + return totalValue; + } else { + if (!flagIndex) { + return totalValue * StorageMultipleConstant; + } + return totalValue; + } + }, count); +}; +/** + * get integet and unit in hard storage and used storage;and the unit of used storage <= the unit of hard storage + * @param hardNumber hard storage number + * @param quotaUnitsDeep clone(Quotas) + * @param usedNumber used storage number + * @param quotaUnitsDeepClone clone(Quotas) + */ +export const GetIntegerAndUnit = (hardNumber: number, quotaUnitsDeep: QuotaUnitInterface[] + , usedNumber: number, quotaUnitsDeepClone: QuotaUnitInterface[]) => { + + for (let unitObj of quotaUnitsDeep) { + if (hardNumber % StorageMultipleConstant === 0 && quotaUnitsDeep.length > 1) { + quotaUnitsDeep.shift(); + if (usedNumber / StorageMultipleConstant >= 1) { + quotaUnitsDeepClone.shift(); + return GetIntegerAndUnit(hardNumber / StorageMultipleConstant + , quotaUnitsDeep, usedNumber / StorageMultipleConstant, quotaUnitsDeepClone); + } else { + return GetIntegerAndUnit(hardNumber / StorageMultipleConstant, quotaUnitsDeep, usedNumber, quotaUnitsDeepClone); + } + } else { + return { + partNumberHard: +hardNumber, + partCharacterHard: unitObj.UNIT, + partNumberUsed: roundDecimals(+usedNumber, 2), + partCharacterUsed: quotaUnitsDeepClone[0].UNIT + }; + } + } +}; +export const validateLimit = (unitContrl) => { + return (control: AbstractControl) => { + if (getByte(control.value, unitContrl.value) > StorageMultipleConstant * StorageMultipleConstant + * StorageMultipleConstant * StorageMultipleConstant * StorageMultipleConstant) { + return { + error: true + }; + } + return null; + }; +}; + diff --git a/src/portal/package.json b/src/portal/package.json index 7112b41e4..79e8c2e37 100644 --- a/src/portal/package.json +++ b/src/portal/package.json @@ -76,7 +76,7 @@ "karma-chrome-launcher": "~2.2.0", "karma-cli": "^1.0.1", "karma-coverage-istanbul-reporter": "~2.0.0", - "karma-jasmine": "^1.1.2", + "karma-jasmine": "^2.0.0", "karma-jasmine-html-reporter": "^0.2.2", "karma-mocha-reporter": "^2.2.4", "karma-remap-istanbul": "^0.6.0", diff --git a/src/portal/src/app/config/config.component.html b/src/portal/src/app/config/config.component.html index 1b89bb2b5..a56f0adf4 100644 --- a/src/portal/src/app/config/config.component.html +++ b/src/portal/src/app/config/config.component.html @@ -32,6 +32,12 @@ + + + + + + \ No newline at end of file diff --git a/src/portal/src/app/harbor-routing.module.ts b/src/portal/src/app/harbor-routing.module.ts index 8d8e10eb9..26ec644df 100644 --- a/src/portal/src/app/harbor-routing.module.ts +++ b/src/portal/src/app/harbor-routing.module.ts @@ -56,6 +56,7 @@ import { ListChartsComponent } from './project/helm-chart/list-charts.component' import { ListChartVersionsComponent } from './project/helm-chart/list-chart-versions/list-chart-versions.component'; import { HelmChartDetailComponent } from './project/helm-chart/helm-chart-detail/chart-detail.component'; import { OidcOnboardComponent } from './oidc-onboard/oidc-onboard.component'; +import { SummaryComponent } from './project/summary/summary.component'; const harborRoutes: Routes = [ { path: '', redirectTo: 'harbor', pathMatch: 'full' }, @@ -165,6 +166,10 @@ const harborRoutes: Routes = [ projectResolver: ProjectRoutingResolver }, children: [ + { + path: 'summary', + component: SummaryComponent + }, { path: 'repositories', component: RepositoryPageComponent diff --git a/src/portal/src/app/project/create-project/create-project.component.html b/src/portal/src/app/project/create-project/create-project.component.html index 3b285000e..e3fa55024 100644 --- a/src/portal/src/app/project/create-project/create-project.component.html +++ b/src/portal/src/app/project/create-project/create-project.component.html @@ -33,6 +33,52 @@ + +
+ + +
+ + + {{'PROJECT.QUOTA_UNLIMIT_TIP' | translate }} + +
+
+
+ + + +
+ + + {{'PROJECT.QUOTA_UNLIMIT_TIP' | translate }} + +
+
diff --git a/src/portal/src/app/project/create-project/create-project.component.ts b/src/portal/src/app/project/create-project/create-project.component.ts index 68bf3140f..eff57e473 100644 --- a/src/portal/src/app/project/create-project/create-project.component.ts +++ b/src/portal/src/app/project/create-project/create-project.component.ts @@ -1,5 +1,5 @@ -import {debounceTime} from 'rxjs/operators'; +import {debounceTime, distinctUntilChanged} from 'rxjs/operators'; // Copyright (c) 2017 VMware, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,9 +19,12 @@ import { Output, ViewChild, OnInit, - OnDestroy + OnDestroy, + Input, + OnChanges, + SimpleChanges } from "@angular/core"; -import { NgForm } from "@angular/forms"; +import { NgForm, Validators, AbstractControl } from "@angular/forms"; import { Subject } from "rxjs"; import { TranslateService } from "@ngx-translate/core"; @@ -30,24 +33,29 @@ import { MessageHandlerService } from "../../shared/message-handler/message-hand import { InlineAlertComponent } from "../../shared/inline-alert/inline-alert.component"; import { Project } from "../project"; -import { ProjectService } from "@harbor/ui"; +import { ProjectService, QuotaUnits, QuotaHardInterface, QuotaUnlimited, getByte + , GetIntegerAndUnit, clone, StorageMultipleConstant, validateLimit} from "@harbor/ui"; import { errorHandler } from '@angular/platform-browser/src/browser'; - - @Component({ selector: "create-project", templateUrl: "create-project.component.html", styleUrls: ["create-project.scss"] }) -export class CreateProjectComponent implements OnInit, OnDestroy { +export class CreateProjectComponent implements OnInit, OnChanges, OnDestroy { projectForm: NgForm; @ViewChild("projectForm") currentForm: NgForm; - + quotaUnits = QuotaUnits; project: Project = new Project(); + countLimit: number; + storageLimit: number; + storageLimitUnit: string = QuotaUnits[3].UNIT; + storageDefaultLimit: number; + storageDefaultLimitUnit: string; + countDefaultLimit: number; initVal: Project = new Project(); createProjectOpened: boolean; @@ -64,6 +72,8 @@ export class CreateProjectComponent implements OnInit, OnDestroy { proNameChecker: Subject = new Subject(); @Output() create = new EventEmitter(); + @Input() quotaObj: QuotaHardInterface; + @Input() isSystemAdmin: boolean; @ViewChild(InlineAlertComponent) inlineAlert: InlineAlertComponent; @@ -97,6 +107,35 @@ export class CreateProjectComponent implements OnInit, OnDestroy { }); } + ngOnChanges(changes: SimpleChanges): void { + if (changes && changes["quotaObj"] && changes["quotaObj"].currentValue) { + this.countLimit = this.quotaObj.count_per_project; + this.storageLimit = GetIntegerAndUnit(this.quotaObj.storage_per_project, clone(QuotaUnits), 0, clone(QuotaUnits)).partNumberHard; + this.storageLimitUnit = this.storageLimit === QuotaUnlimited ? QuotaUnits[3].UNIT + : GetIntegerAndUnit(this.quotaObj.storage_per_project, clone(QuotaUnits), 0, clone(QuotaUnits)).partCharacterHard; + + this.countDefaultLimit = this.countLimit; + this.storageDefaultLimit = this.storageLimit; + this.storageDefaultLimitUnit = this.storageLimitUnit; + if (this.isSystemAdmin) { + this.currentForm.form.controls['create_project_storage-limit'].setValidators( + [ + Validators.required, + Validators.pattern('(^-1$)|(^([1-9]+)([0-9]+)*$)'), + validateLimit(this.currentForm.form.controls['create_project_storage-limit-unit']) + ]); + } + this.currentForm.form.valueChanges + .pipe(distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b))) + .subscribe((data) => { + ['create_project_storage-limit', 'create_project_storage-limit-unit'].forEach(fieldName => { + if (this.currentForm.form.get(fieldName) && this.currentForm.form.get(fieldName).value !== null) { + this.currentForm.form.get(fieldName).updateValueAndValidity(); + } + }); + }); + } +} ngOnDestroy(): void { this.proNameChecker.unsubscribe(); } @@ -105,10 +144,10 @@ export class CreateProjectComponent implements OnInit, OnDestroy { if (this.isSubmitOnGoing) { return ; } - this.isSubmitOnGoing = true; + const storageByte = +this.storageLimit === QuotaUnlimited ? this.storageLimit : getByte(+this.storageLimit, this.storageLimitUnit); this.projectService - .createProject(this.project.name, this.project.metadata) + .createProject(this.project.name, this.project.metadata, +this.countLimit, +storageByte) .subscribe( status => { this.isSubmitOnGoing = false; @@ -127,7 +166,6 @@ export class CreateProjectComponent implements OnInit, OnDestroy { this.createProjectOpened = false; } - newProject() { this.project = new Project(); this.hasChanged = false; @@ -135,6 +173,10 @@ export class CreateProjectComponent implements OnInit, OnDestroy { this.createProjectOpened = true; this.inlineAlert.close(); + + this.countLimit = this.countDefaultLimit ; + this.storageLimit = this.storageDefaultLimit; + this.storageLimitUnit = this.storageDefaultLimitUnit; } public get isValid(): boolean { diff --git a/src/portal/src/app/project/create-project/create-project.scss b/src/portal/src/app/project/create-project/create-project.scss index 8661ba903..ec7820ed9 100644 --- a/src/portal/src/app/project/create-project/create-project.scss +++ b/src/portal/src/app/project/create-project/create-project.scss @@ -17,7 +17,7 @@ .form-block > div { padding-left: 135px; .input-width { - width: 296px; + width: 196px; } .public-tooltip { top: -8px; @@ -29,3 +29,19 @@ } } + +.form-group { + ::ng-deep { + clr-select-container { + margin-top: 0.3rem; + } + } + select { + display: inline; + } + .checkbox-inline { + margin-left: 5px; + height: 1rem; + } +} + diff --git a/src/portal/src/app/project/list-project/list-project.component.ts b/src/portal/src/app/project/list-project/list-project.component.ts index dd9c64a79..59477512d 100644 --- a/src/portal/src/app/project/list-project/list-project.component.ts +++ b/src/portal/src/app/project/list-project/list-project.component.ts @@ -143,7 +143,7 @@ export class ListProjectComponent implements OnDestroy { goToLink(proId: number): void { this.searchTrigger.closeSearch(true); - let linkUrl = ["harbor", "projects", proId, "repositories"]; + let linkUrl = ["harbor", "projects", proId, "summary"]; this.router.navigate(linkUrl); } diff --git a/src/portal/src/app/project/project-detail/project-detail.component.html b/src/portal/src/app/project/project-detail/project-detail.component.html index 89820aa8a..0589d5138 100644 --- a/src/portal/src/app/project/project-detail/project-detail.component.html +++ b/src/portal/src/app/project/project-detail/project-detail.component.html @@ -4,6 +4,9 @@

{{currentProject.name}} {{roleName | translate}}