From 24c375358185fe540a0ad5e5160dca23fd60fad9 Mon Sep 17 00:00:00 2001 From: wang yan Date: Wed, 3 Jul 2019 15:47:45 +0800 Subject: [PATCH] add dao of artifact Signed-off-by: wang yan Add dao for quota Signed-off-by: He Weiwei fix govet Signed-off-by: wang yan --- .../postgresql/0010_1.9.0_schema.up.sql | 64 +++++++- src/common/dao/artifact.go | 60 ++++++++ src/common/dao/artifact_blob.go | 107 +++++++++++++ src/common/dao/artifact_blob_test.go | 131 ++++++++++++++++ src/common/dao/artifact_test.go | 92 +++++++++++ src/common/dao/blob.go | 50 ++++++ src/common/dao/blob_test.go | 65 ++++++++ src/common/dao/quota.go | 144 ++++++++++++++++++ src/common/dao/quota_test.go | 135 ++++++++++++++++ src/common/dao/quota_usage.go | 144 ++++++++++++++++++ src/common/dao/quota_usage_test.go | 135 ++++++++++++++++ src/common/models/artifact.go | 23 +++ src/common/models/artifact_blob.go | 18 +++ src/common/models/base.go | 8 +- src/common/models/blob.go | 19 +++ src/common/models/quota.go | 77 ++++++++++ src/common/models/quota_usage.go | 77 ++++++++++ 17 files changed, 1347 insertions(+), 2 deletions(-) create mode 100644 src/common/dao/artifact.go create mode 100644 src/common/dao/artifact_blob.go create mode 100644 src/common/dao/artifact_blob_test.go create mode 100644 src/common/dao/artifact_test.go create mode 100644 src/common/dao/blob.go create mode 100644 src/common/dao/blob_test.go create mode 100644 src/common/dao/quota.go create mode 100644 src/common/dao/quota_test.go create mode 100644 src/common/dao/quota_usage.go create mode 100644 src/common/dao/quota_usage_test.go create mode 100644 src/common/models/artifact.go create mode 100644 src/common/models/artifact_blob.go create mode 100644 src/common/models/blob.go create mode 100644 src/common/models/quota.go create mode 100644 src/common/models/quota_usage.go diff --git a/make/migrations/postgresql/0010_1.9.0_schema.up.sql b/make/migrations/postgresql/0010_1.9.0_schema.up.sql index 11c2248a6..afbf37a3b 100644 --- a/make/migrations/postgresql/0010_1.9.0_schema.up.sql +++ b/make/migrations/postgresql/0010_1.9.0_schema.up.sql @@ -7,4 +7,66 @@ CREATE TABLE cve_whitelist ( expires_at bigint, items text NOT NULL, UNIQUE (project_id) -); \ No newline at end of file +); + +CREATE TABLE blob ( + id SERIAL PRIMARY KEY NOT NULL, + /* + digest of config, layer, manifest + */ + digest varchar(255) NOT NULL, + content_type varchar(255) NOT NULL, + size int NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + UNIQUE (digest) +); + +CREATE TABLE artifact ( + id SERIAL PRIMARY KEY NOT NULL, + project_id int NOT NULL, + repo varchar(255) NOT NULL, + tag varchar(255) NOT NULL, + /* + digest of manifest + */ + digest varchar(255) NOT NULL, + /* + kind of artifact, image, chart, etc.. + */ + kind varchar(255) NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + pull_time timestamp, + push_time timestamp, + CONSTRAINT unique_artifact UNIQUE (project_id, repo, tag) +); + +/* add the table for relation of artifact and blob */ +CREATE TABLE artifact_blob ( + id SERIAL PRIMARY KEY NOT NULL, + digest_af varchar(255) NOT NULL, + digest_blob varchar(255) NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + CONSTRAINT unique_artifact_blob UNIQUE (digest_af, digest_blob) +); + +/* add quota table */ +CREATE TABLE quota ( + id SERIAL PRIMARY KEY NOT NULL, + reference VARCHAR(255) NOT NULL, + reference_id VARCHAR(255) NOT NULL, + hard JSONB NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP, + UNIQUE(reference, reference_id) +); + +/* add quota usage table */ +CREATE TABLE quota_usage ( + id SERIAL PRIMARY KEY NOT NULL, + reference VARCHAR(255) NOT NULL, + reference_id VARCHAR(255) NOT NULL, + used JSONB NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP, + UNIQUE(reference, reference_id) +); diff --git a/src/common/dao/artifact.go b/src/common/dao/artifact.go new file mode 100644 index 000000000..d8ba7d86c --- /dev/null +++ b/src/common/dao/artifact.go @@ -0,0 +1,60 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "github.com/goharbor/harbor/src/common/models" + "strings" + "time" +) + +// AddArtifact ... +func AddArtifact(af *models.Artifact) (int64, error) { + now := time.Now() + af.CreationTime = now + id, err := GetOrmer().Insert(af) + if err != nil { + if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + return 0, ErrDupRows + } + return 0, err + } + return id, nil +} + +// DeleteArtifact ... +func DeleteArtifact(id int64) error { + _, err := GetOrmer().QueryTable(&models.Artifact{}).Filter("ID", id).Delete() + return err +} + +// DeleteArtifactByDigest ... +func DeleteArtifactByDigest(digest string) error { + _, err := GetOrmer().Raw(`delete from artifact where digest = ? `, digest).Exec() + if err != nil { + return err + } + return nil +} + +// DeleteByTag ... +func DeleteByTag(projectID int, repo, tag string) error { + _, err := GetOrmer().Raw(`delete from artifact where project_id = ? and repo = ? and tag = ? `, + projectID, repo, tag).Exec() + if err != nil { + return err + } + return nil +} diff --git a/src/common/dao/artifact_blob.go b/src/common/dao/artifact_blob.go new file mode 100644 index 000000000..e0f0f089e --- /dev/null +++ b/src/common/dao/artifact_blob.go @@ -0,0 +1,107 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "fmt" + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/pkg/errors" + "strconv" + "strings" + "time" +) + +// AddArtifactNBlob ... +func AddArtifactNBlob(afnb *models.ArtifactAndBlob) (int64, error) { + now := time.Now() + afnb.CreationTime = now + id, err := GetOrmer().Insert(afnb) + if err != nil { + if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + return 0, ErrDupRows + } + return 0, err + } + return id, nil +} + +// AddArtifactNBlobs ... +func AddArtifactNBlobs(afnbs []*models.ArtifactAndBlob) error { + o := orm.NewOrm() + err := o.Begin() + if err != nil { + return err + } + + var errInsertMultiple error + total := len(afnbs) + successNums, err := o.InsertMulti(total, afnbs) + if err != nil { + errInsertMultiple = err + err := o.Rollback() + if err != nil { + log.Errorf("fail to rollback when to insert multiple artifact and blobs, %v", err) + errInsertMultiple = errors.Wrap(errInsertMultiple, err.Error()) + } + return errInsertMultiple + } + + // part of them cannot be inserted successfully. + if successNums != int64(total) { + errInsertMultiple = errors.New("Not all of artifact and blobs are inserted successfully") + err := o.Rollback() + if err != nil { + log.Errorf("fail to rollback when to insert multiple artifact and blobs, %v", err) + errInsertMultiple = errors.Wrap(errInsertMultiple, err.Error()) + } + return errInsertMultiple + } + + err = o.Commit() + if err != nil { + log.Errorf("fail to commit when to insert multiple artifact and blobs, %v", err) + return fmt.Errorf("fail to commit when to insert multiple artifact and blobs, %v", err) + } + + return nil +} + +// DeleteArtifactAndBlobByDigest ... +func DeleteArtifactAndBlobByDigest(digest string) error { + _, err := GetOrmer().Raw(`delete from artifact_blob where digest_af = ? `, digest).Exec() + if err != nil { + return err + } + return nil +} + +// CountSizeOfArtifact ... +func CountSizeOfArtifact(digest string) (int64, error) { + var res []orm.Params + num, err := GetOrmer().Raw(`SELECT sum(bb.size) FROM artifact_blob afnb LEFT JOIN blob bb ON afnb.digest_blob = bb.digest WHERE afnb.digest_af = ? `, digest).Values(&res) + if err != nil { + return -1, err + } + if num > 0 { + size, err := strconv.ParseInt(res[0]["sum"].(string), 0, 64) + if err != nil { + return -1, err + } + return size, nil + } + return -1, err +} diff --git a/src/common/dao/artifact_blob_test.go b/src/common/dao/artifact_blob_test.go new file mode 100644 index 000000000..3da44748b --- /dev/null +++ b/src/common/dao/artifact_blob_test.go @@ -0,0 +1,131 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "testing" + + "github.com/goharbor/harbor/src/common/models" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAddArtifactNBlob(t *testing.T) { + afnb := &models.ArtifactAndBlob{ + DigestAF: "vvvv", + DigestBlob: "aaaa", + } + + // add + id, err := AddArtifactNBlob(afnb) + require.Nil(t, err) + afnb.ID = id + assert.Equal(t, id, int64(1)) +} + +func TestAddArtifactNBlobs(t *testing.T) { + afnb1 := &models.ArtifactAndBlob{ + DigestAF: "zzzz", + DigestBlob: "zzza", + } + afnb2 := &models.ArtifactAndBlob{ + DigestAF: "zzzz", + DigestBlob: "zzzb", + } + afnb3 := &models.ArtifactAndBlob{ + DigestAF: "zzzz", + DigestBlob: "zzzc", + } + + var afnbs []*models.ArtifactAndBlob + afnbs = append(afnbs, afnb1) + afnbs = append(afnbs, afnb2) + afnbs = append(afnbs, afnb3) + + // add + err := AddArtifactNBlobs(afnbs) + require.Nil(t, err) +} + +func TestDeleteArtifactAndBlobByDigest(t *testing.T) { + afnb := &models.ArtifactAndBlob{ + DigestAF: "vvvv", + DigestBlob: "vvva", + } + + // add + _, err := AddArtifactNBlob(afnb) + require.Nil(t, err) + + // delete + err = DeleteArtifactAndBlobByDigest(afnb.DigestAF) + require.Nil(t, err) +} + +func TestCountSizeOfArtifact(t *testing.T) { + + afnb1 := &models.ArtifactAndBlob{ + DigestAF: "xxxx", + DigestBlob: "aaaa", + } + afnb2 := &models.ArtifactAndBlob{ + DigestAF: "xxxx", + DigestBlob: "aaab", + } + afnb3 := &models.ArtifactAndBlob{ + DigestAF: "xxxx", + DigestBlob: "aaac", + } + + var afnbs []*models.ArtifactAndBlob + afnbs = append(afnbs, afnb1) + afnbs = append(afnbs, afnb2) + afnbs = append(afnbs, afnb3) + + err := AddArtifactNBlobs(afnbs) + require.Nil(t, err) + + blob1 := &models.Blob{ + Digest: "aaaa", + ContentType: "v2.blob", + Size: 100, + } + + _, err = AddBlob(blob1) + require.Nil(t, err) + + blob2 := &models.Blob{ + Digest: "aaab", + ContentType: "v2.blob", + Size: 200, + } + + _, err = AddBlob(blob2) + require.Nil(t, err) + + blob3 := &models.Blob{ + Digest: "aaac", + ContentType: "v2.blob", + Size: 300, + } + + _, err = AddBlob(blob3) + require.Nil(t, err) + + imageSize, err := CountSizeOfArtifact("xxxx") + require.Nil(t, err) + require.Equal(t, imageSize, int64(600)) +} diff --git a/src/common/dao/artifact_test.go b/src/common/dao/artifact_test.go new file mode 100644 index 000000000..5c9405087 --- /dev/null +++ b/src/common/dao/artifact_test.go @@ -0,0 +1,92 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "testing" + + "github.com/goharbor/harbor/src/common/models" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAddArtifact(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "hello-world", + Tag: "latest", + Digest: "1234abcd", + Kind: "image", + } + + // add + id, err := AddArtifact(af) + require.Nil(t, err) + af.ID = id + assert.Equal(t, id, int64(1)) + +} + +func TestDeleteArtifact(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "hello-world", + Tag: "v1.0", + Digest: "1234abcd", + Kind: "image", + } + // add + id, err := AddArtifact(af) + require.Nil(t, err) + + // delete + err = DeleteArtifact(id) + require.Nil(t, err) +} + +func TestDeleteArtifactByDigest(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "hello-world", + Tag: "v1.1", + Digest: "TestDeleteArtifactByDigest", + Kind: "image", + } + // add + _, err := AddArtifact(af) + require.Nil(t, err) + + // delete + err = DeleteArtifactByDigest(af.Digest) + require.Nil(t, err) +} + +func TestDeleteArtifactByTag(t *testing.T) { + af := &models.Artifact{ + PID: 1, + Repo: "hello-world", + Tag: "v1.2", + Digest: "TestDeleteArtifactByTag", + Kind: "image", + } + // add + _, err := AddArtifact(af) + require.Nil(t, err) + + // delete + err = DeleteByTag(1, "hello-world", "v1.2") + require.Nil(t, err) +} diff --git a/src/common/dao/blob.go b/src/common/dao/blob.go new file mode 100644 index 000000000..e63d218d2 --- /dev/null +++ b/src/common/dao/blob.go @@ -0,0 +1,50 @@ +package dao + +import ( + "fmt" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + "strings" + "time" +) + +// AddBlob ... +func AddBlob(blob *models.Blob) (int64, error) { + now := time.Now() + blob.CreationTime = now + id, err := GetOrmer().Insert(blob) + if err != nil { + if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + return 0, ErrDupRows + } + return 0, err + } + return id, nil +} + +// GetBlob ... +func GetBlob(digest string) (*models.Blob, error) { + o := GetOrmer() + qs := o.QueryTable(&models.Blob{}) + qs = qs.Filter("Digest", digest) + b := []*models.Blob{} + _, err := qs.All(&b) + if err != nil { + return nil, fmt.Errorf("failed to get blob for digest %s, error: %v", digest, err) + } + if len(b) == 0 { + log.Infof("No blob found for digest %s, returning empty.", digest) + return &models.Blob{}, nil + } else if len(b) > 1 { + log.Infof("Multiple blob found for digest %s", digest) + return &models.Blob{}, fmt.Errorf("Multiple blob found for digest %s", digest) + } + return b[0], nil +} + +// DeleteBlob ... +func DeleteBlob(digest string) error { + o := GetOrmer() + _, err := o.QueryTable("blob").Filter("digest", digest).Delete() + return err +} diff --git a/src/common/dao/blob_test.go b/src/common/dao/blob_test.go new file mode 100644 index 000000000..0ae70ff5c --- /dev/null +++ b/src/common/dao/blob_test.go @@ -0,0 +1,65 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "github.com/goharbor/harbor/src/common/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestAddBlob(t *testing.T) { + blob := &models.Blob{ + Digest: "1234abcd", + ContentType: "v2.blob", + Size: 1523, + } + + // add + _, err := AddBlob(blob) + require.Nil(t, err) +} + +func TestGetBlob(t *testing.T) { + blob := &models.Blob{ + Digest: "12345abcde", + ContentType: "v2.blob", + Size: 453, + } + + // add + id, err := AddBlob(blob) + require.Nil(t, err) + blob.ID = id + + blob2, err := GetBlob("12345abcde") + require.Nil(t, err) + assert.Equal(t, blob.Digest, blob2.Digest) + +} + +func TestDeleteBlob(t *testing.T) { + blob := &models.Blob{ + Digest: "123456abcdef", + ContentType: "v2.blob", + Size: 4543, + } + id, err := AddBlob(blob) + require.Nil(t, err) + blob.ID = id + err = DeleteBlob(blob.Digest) + require.Nil(t, err) +} diff --git a/src/common/dao/quota.go b/src/common/dao/quota.go new file mode 100644 index 000000000..4252e51e6 --- /dev/null +++ b/src/common/dao/quota.go @@ -0,0 +1,144 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "fmt" + "strings" + "time" + + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/models" +) + +var ( + quotaOrderMap = map[string]string{ + "id": "id asc", + "+id": "id asc", + "-id": "id desc", + "creation_time": "creation_time asc", + "+creation_time": "creation_time asc", + "-creation_time": "creation_time desc", + "update_time": "update_time asc", + "+update_time": "update_time asc", + "-update_time": "update_time desc", + } +) + +// AddQuota add quota to the database. +func AddQuota(quota models.Quota) (int64, error) { + now := time.Now() + quota.CreationTime = now + quota.UpdateTime = now + return GetOrmer().Insert("a) +} + +// GetQuota returns quota by id. +func GetQuota(id int64) (*models.Quota, error) { + q := models.Quota{ID: id} + err := GetOrmer().Read(&q, "ID") + if err == orm.ErrNoRows { + return nil, nil + } + return &q, err +} + +// UpdateQuota update the quota. +func UpdateQuota(quota models.Quota) error { + quota.UpdateTime = time.Now() + _, err := GetOrmer().Update("a) + return err +} + +// ListQuotas returns quotas by query. +func ListQuotas(query ...*models.QuotaQuery) ([]*models.Quota, error) { + condition, params := quotaQueryConditions(query...) + sql := fmt.Sprintf(`select * %s`, condition) + + orderBy := quotaOrderBy(query...) + if orderBy != "" { + sql += ` order by ` + orderBy + } + + if len(query) > 0 && query[0] != nil { + page, size := query[0].Page, query[0].Size + if size > 0 { + sql += ` limit ?` + params = append(params, size) + if page > 0 { + sql += ` offset ?` + params = append(params, size*(page-1)) + } + } + } + + var quotas []*models.Quota + if _, err := GetOrmer().Raw(sql, params).QueryRows("as); err != nil { + return nil, err + } + + return quotas, nil +} + +func quotaQueryConditions(query ...*models.QuotaQuery) (string, []interface{}) { + params := []interface{}{} + sql := `from quota ` + if len(query) == 0 || query[0] == nil { + return sql, params + } + + sql += `where 1=1 ` + + q := query[0] + if q.Reference != "" { + sql += `and reference = ? ` + params = append(params, q.Reference) + } + if q.ReferenceID != "" { + sql += `and reference_id = ? ` + params = append(params, q.ReferenceID) + } + if len(q.ReferenceIDs) != 0 { + sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs))) + params = append(params, q.ReferenceIDs) + } + + return sql, params +} + +func quotaOrderBy(query ...*models.QuotaQuery) string { + orderBy := "" + + if len(query) > 0 && query[0] != nil && query[0].Sort != "" { + if val, ok := quotaOrderMap[query[0].Sort]; ok { + orderBy = val + } else { + sort := query[0].Sort + + order := "asc" + if sort[0] == '-' { + order = "desc" + sort = sort[1:] + } + + prefix := "hard." + if strings.HasPrefix(sort, prefix) { + orderBy = fmt.Sprintf("hard->>'%s' %s", strings.TrimPrefix(sort, prefix), order) + } + } + } + + return orderBy +} diff --git a/src/common/dao/quota_test.go b/src/common/dao/quota_test.go new file mode 100644 index 000000000..9ea2185d5 --- /dev/null +++ b/src/common/dao/quota_test.go @@ -0,0 +1,135 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "testing" + "time" + + "github.com/goharbor/harbor/src/common/models" + "github.com/stretchr/testify/suite" +) + +var ( + quotaReference = "project" + quotaUserReference = "user" + quotaHard = models.QuotaHard{"storage": 1024} + quotaHardLarger = models.QuotaHard{"storage": 2048} +) + +type QuotaDaoSuite struct { + suite.Suite +} + +func (suite *QuotaDaoSuite) equalHard(quota1 *models.Quota, quota2 *models.Quota) { + hard1, err := quota1.GetHard() + suite.Nil(err, "hard1 invalid") + + hard2, err := quota2.GetHard() + suite.Nil(err, "hard2 invalid") + + suite.Equal(hard1, hard2) +} + +func (suite *QuotaDaoSuite) TearDownTest() { + ClearTable("quota") +} + +func (suite *QuotaDaoSuite) TestAddQuota() { + _, err1 := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}) + suite.Nil(err1) + + // Will failed for reference and reference_id should unique in db + _, err2 := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}) + suite.Error(err2) + + _, err3 := AddQuota(models.Quota{Reference: quotaUserReference, ReferenceID: "1", Hard: quotaHard.String()}) + suite.Nil(err3) +} + +func (suite *QuotaDaoSuite) TestGetQuota() { + quota1 := models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()} + id, err := AddQuota(quota1) + suite.Nil(err) + + // Get the new added quota + quota2, err := GetQuota(id) + suite.Nil(err) + suite.NotNil(quota2) + + // Get the quota which id is 10000 not found + quota3, err := GetQuota(10000) + suite.Nil(err) + suite.Nil(quota3) +} + +func (suite *QuotaDaoSuite) TestUpdateQuota() { + quota1 := models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()} + id, err := AddQuota(quota1) + suite.Nil(err) + + // Get the new added quota + quota2, err := GetQuota(id) + suite.Nil(err) + suite.equalHard("a1, quota2) + + // Update the quota + quota2.SetHard(quotaHardLarger) + time.Sleep(time.Millisecond * 10) // Ensure that UpdateTime changed + suite.Nil(UpdateQuota(*quota2)) + + // Get the updated quota + quota3, err := GetQuota(id) + suite.Nil(err) + suite.equalHard(quota2, quota3) + suite.NotEqual(quota2.UpdateTime, quota3.UpdateTime) +} + +func (suite *QuotaDaoSuite) TestListQuotas() { + AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}) + AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "2", Hard: quotaHard.String()}) + AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "3", Hard: quotaHard.String()}) + AddQuota(models.Quota{Reference: quotaUserReference, ReferenceID: "1", Hard: quotaHardLarger.String()}) + + // List all the quotas + quotas, err := ListQuotas() + suite.Nil(err) + suite.Equal(4, len(quotas)) + suite.Equal(quotaReference, quotas[0].Reference) + + // List quotas filter by reference + quotas, err = ListQuotas(&models.QuotaQuery{Reference: quotaReference}) + suite.Nil(err) + suite.Equal(3, len(quotas)) + + // List quotas filter by reference ids + quotas, err = ListQuotas(&models.QuotaQuery{Reference: quotaReference, ReferenceIDs: []string{"1", "2"}}) + suite.Nil(err) + suite.Equal(2, len(quotas)) + + // List quotas by pagination + quotas, err = ListQuotas(&models.QuotaQuery{Pagination: models.Pagination{Size: 2}}) + suite.Nil(err) + suite.Equal(2, len(quotas)) + + // List quotas by sorting + quotas, err = ListQuotas(&models.QuotaQuery{Sorting: models.Sorting{Sort: "-hard.storage"}}) + suite.Nil(err) + suite.Equal(quotaUserReference, quotas[0].Reference) +} + +func TestRunQuotaDaoSuite(t *testing.T) { + suite.Run(t, new(QuotaDaoSuite)) +} diff --git a/src/common/dao/quota_usage.go b/src/common/dao/quota_usage.go new file mode 100644 index 000000000..8e2f7ca48 --- /dev/null +++ b/src/common/dao/quota_usage.go @@ -0,0 +1,144 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "fmt" + "strings" + "time" + + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/models" +) + +var ( + quotaUsageOrderMap = map[string]string{ + "id": "id asc", + "+id": "id asc", + "-id": "id desc", + "creation_time": "creation_time asc", + "+creation_time": "creation_time asc", + "-creation_time": "creation_time desc", + "update_time": "update_time asc", + "+update_time": "update_time asc", + "-update_time": "update_time desc", + } +) + +// AddQuotaUsage add quota usage to the database. +func AddQuotaUsage(quotaUsage models.QuotaUsage) (int64, error) { + now := time.Now() + quotaUsage.CreationTime = now + quotaUsage.UpdateTime = now + return GetOrmer().Insert("aUsage) +} + +// GetQuotaUsage returns quota usage by id. +func GetQuotaUsage(id int64) (*models.QuotaUsage, error) { + q := models.QuotaUsage{ID: id} + err := GetOrmer().Read(&q, "ID") + if err == orm.ErrNoRows { + return nil, nil + } + return &q, err +} + +// UpdateQuotaUsage update the quota usage. +func UpdateQuotaUsage(quotaUsage models.QuotaUsage) error { + quotaUsage.UpdateTime = time.Now() + _, err := GetOrmer().Update("aUsage) + return err +} + +// ListQuotaUsages returns quota usages by query. +func ListQuotaUsages(query ...*models.QuotaUsageQuery) ([]*models.QuotaUsage, error) { + condition, params := quotaUsageQueryConditions(query...) + sql := fmt.Sprintf(`select * %s`, condition) + + orderBy := quotaUsageOrderBy(query...) + if orderBy != "" { + sql += ` order by ` + orderBy + } + + if len(query) > 0 && query[0] != nil { + page, size := query[0].Page, query[0].Size + if size > 0 { + sql += ` limit ?` + params = append(params, size) + if page > 0 { + sql += ` offset ?` + params = append(params, size*(page-1)) + } + } + } + + var quotaUsages []*models.QuotaUsage + if _, err := GetOrmer().Raw(sql, params).QueryRows("aUsages); err != nil { + return nil, err + } + + return quotaUsages, nil +} + +func quotaUsageQueryConditions(query ...*models.QuotaUsageQuery) (string, []interface{}) { + params := []interface{}{} + sql := `from quota_usage ` + if len(query) == 0 || query[0] == nil { + return sql, params + } + + sql += `where 1=1 ` + + q := query[0] + if q.Reference != "" { + sql += `and reference = ? ` + params = append(params, q.Reference) + } + if q.ReferenceID != "" { + sql += `and reference_id = ? ` + params = append(params, q.ReferenceID) + } + if len(q.ReferenceIDs) != 0 { + sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs))) + params = append(params, q.ReferenceIDs) + } + + return sql, params +} + +func quotaUsageOrderBy(query ...*models.QuotaUsageQuery) string { + orderBy := "" + + if len(query) > 0 && query[0] != nil && query[0].Sort != "" { + if val, ok := quotaUsageOrderMap[query[0].Sort]; ok { + orderBy = val + } else { + sort := query[0].Sort + + order := "asc" + if sort[0] == '-' { + order = "desc" + sort = sort[1:] + } + + prefix := "used." + if strings.HasPrefix(sort, prefix) { + orderBy = fmt.Sprintf("used->>'%s' %s", strings.TrimPrefix(sort, prefix), order) + } + } + } + + return orderBy +} diff --git a/src/common/dao/quota_usage_test.go b/src/common/dao/quota_usage_test.go new file mode 100644 index 000000000..40ff14124 --- /dev/null +++ b/src/common/dao/quota_usage_test.go @@ -0,0 +1,135 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "testing" + "time" + + "github.com/goharbor/harbor/src/common/models" + "github.com/stretchr/testify/suite" +) + +var ( + quotaUsageReference = "project" + quotaUsageUserReference = "user" + quotaUsageUsed = models.QuotaUsed{"storage": 1024} + quotaUsageUsedLarger = models.QuotaUsed{"storage": 2048} +) + +type QuotaUsageDaoSuite struct { + suite.Suite +} + +func (suite *QuotaUsageDaoSuite) equalUsed(usage1 *models.QuotaUsage, usage2 *models.QuotaUsage) { + used1, err := usage1.GetUsed() + suite.Nil(err, "used1 invalid") + + used2, err := usage2.GetUsed() + suite.Nil(err, "used2 invalid") + + suite.Equal(used1, used2) +} + +func (suite *QuotaUsageDaoSuite) TearDownTest() { + ClearTable("quota_usage") +} + +func (suite *QuotaUsageDaoSuite) TestAddQuotaUsage() { + _, err1 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}) + suite.Nil(err1) + + // Will failed for reference and reference_id should unique in db + _, err2 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}) + suite.Error(err2) + + _, err3 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageUserReference, ReferenceID: "1", Used: quotaUsageUsed.String()}) + suite.Nil(err3) +} + +func (suite *QuotaUsageDaoSuite) TestGetQuotaUsage() { + quotaUsage1 := models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()} + id, err := AddQuotaUsage(quotaUsage1) + suite.Nil(err) + + // Get the new added quotaUsage + quotaUsage2, err := GetQuotaUsage(id) + suite.Nil(err) + suite.NotNil(quotaUsage2) + + // Get the quotaUsage which id is 10000 not found + quotaUsage3, err := GetQuotaUsage(10000) + suite.Nil(err) + suite.Nil(quotaUsage3) +} + +func (suite *QuotaUsageDaoSuite) TestUpdateQuotaUsage() { + quotaUsage1 := models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()} + id, err := AddQuotaUsage(quotaUsage1) + suite.Nil(err) + + // Get the new added quotaUsage + quotaUsage2, err := GetQuotaUsage(id) + suite.Nil(err) + suite.equalUsed("aUsage1, quotaUsage2) + + // Update the quotaUsage + quotaUsage2.SetUsed(quotaUsageUsedLarger) + time.Sleep(time.Millisecond * 10) // Ensure that UpdateTime changed + suite.Nil(UpdateQuotaUsage(*quotaUsage2)) + + // Get the updated quotaUsage + quotaUsage3, err := GetQuotaUsage(id) + suite.Nil(err) + suite.equalUsed(quotaUsage2, quotaUsage3) + suite.NotEqual(quotaUsage2.UpdateTime, quotaUsage3.UpdateTime) +} + +func (suite *QuotaUsageDaoSuite) TestListQuotaUsages() { + AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}) + AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "2", Used: quotaUsageUsed.String()}) + AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "3", Used: quotaUsageUsed.String()}) + AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageUserReference, ReferenceID: "1", Used: quotaUsageUsedLarger.String()}) + + // List all the quotaUsages + quotaUsages, err := ListQuotaUsages() + suite.Nil(err) + suite.Equal(4, len(quotaUsages)) + suite.Equal(quotaUsageReference, quotaUsages[0].Reference) + + // List quotaUsages filter by reference + quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Reference: quotaUsageReference}) + suite.Nil(err) + suite.Equal(3, len(quotaUsages)) + + // List quotaUsages filter by reference ids + quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Reference: quotaUsageReference, ReferenceIDs: []string{"1", "2"}}) + suite.Nil(err) + suite.Equal(2, len(quotaUsages)) + + // List quotaUsages by pagination + quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Pagination: models.Pagination{Size: 2}}) + suite.Nil(err) + suite.Equal(2, len(quotaUsages)) + + // List quotaUsages by sorting + quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Sorting: models.Sorting{Sort: "-used.storage"}}) + suite.Nil(err) + suite.Equal(quotaUsageUserReference, quotaUsages[0].Reference) +} + +func TestRunQuotaUsageDaoSuite(t *testing.T) { + suite.Run(t, new(QuotaUsageDaoSuite)) +} diff --git a/src/common/models/artifact.go b/src/common/models/artifact.go new file mode 100644 index 000000000..63abd9760 --- /dev/null +++ b/src/common/models/artifact.go @@ -0,0 +1,23 @@ +package models + +import ( + "time" +) + +// Artifact holds the details of a artifact. +type Artifact struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + PID int64 `orm:"column(project_id)" json:"project_id"` + Repo string `orm:"column(repo)" json:"repo"` + Tag string `orm:"column(tag)" json:"tag"` + Digest string `orm:"column(digest)" json:"digest"` + Kind string `orm:"column(kind)" json:"kind"` + PushTime time.Time `orm:"column(push_time)" json:"push_time"` + PullTime time.Time `orm:"column(pull_time)" json:"pull_time"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` +} + +// TableName ... +func (af *Artifact) TableName() string { + return "artifact" +} diff --git a/src/common/models/artifact_blob.go b/src/common/models/artifact_blob.go new file mode 100644 index 000000000..a402306ee --- /dev/null +++ b/src/common/models/artifact_blob.go @@ -0,0 +1,18 @@ +package models + +import ( + "time" +) + +// ArtifactAndBlob holds the relationship between manifest and blob. +type ArtifactAndBlob struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + DigestAF string `orm:"column(digest_af)" json:"digest_af"` + DigestBlob string `orm:"column(digest_blob)" json:"digest_blob"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` +} + +// TableName ... +func (afb *ArtifactAndBlob) TableName() string { + return "artifact_blob" +} diff --git a/src/common/models/base.go b/src/common/models/base.go index f6d666ee8..7ecee503c 100644 --- a/src/common/models/base.go +++ b/src/common/models/base.go @@ -37,5 +37,11 @@ func init() { new(JobLog), new(Robot), new(OIDCUser), - new(CVEWhitelist)) + new(Blob), + new(Artifact), + new(ArtifactAndBlob), + new(CVEWhitelist), + new(Quota), + new(QuotaUsage), + ) } diff --git a/src/common/models/blob.go b/src/common/models/blob.go new file mode 100644 index 000000000..71a3c9b67 --- /dev/null +++ b/src/common/models/blob.go @@ -0,0 +1,19 @@ +package models + +import ( + "time" +) + +// Blob holds the details of a blob. +type Blob struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Digest string `orm:"column(digest)" json:"digest"` + ContentType string `orm:"column(content_type)" json:"content_type"` + Size int64 `orm:"column(size)" json:"size"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` +} + +// TableName ... +func (b *Blob) TableName() string { + return "blob" +} diff --git a/src/common/models/quota.go b/src/common/models/quota.go new file mode 100644 index 000000000..4b83bb338 --- /dev/null +++ b/src/common/models/quota.go @@ -0,0 +1,77 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "encoding/json" + "time" +) + +// QuotaHard a map for the quota hard +type QuotaHard map[string]int64 + +func (h QuotaHard) String() string { + bytes, _ := json.Marshal(h) + return string(bytes) +} + +// Copy returns copied quota hard +func (h QuotaHard) Copy() QuotaHard { + hard := QuotaHard{} + for key, value := range h { + hard[key] = value + } + + return hard +} + +// Quota model for quota +type Quota struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Reference string `orm:"column(reference)" json:"reference"` + ReferenceID string `orm:"column(reference_id)" json:"reference_id"` + Hard string `orm:"column(hard);type(jsonb)" json:"-"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +// TableName returns table name for orm +func (q *Quota) TableName() string { + return "quota" +} + +// GetHard returns quota hard +func (q *Quota) GetHard() (QuotaHard, error) { + var hard QuotaHard + if err := json.Unmarshal([]byte(q.Hard), &hard); err != nil { + return nil, err + } + + return hard, nil +} + +// SetHard set new quota hard +func (q *Quota) SetHard(hard QuotaHard) { + q.Hard = hard.String() +} + +// QuotaQuery query parameters for quota +type QuotaQuery struct { + Reference string + ReferenceID string + ReferenceIDs []string + Pagination + Sorting +} diff --git a/src/common/models/quota_usage.go b/src/common/models/quota_usage.go new file mode 100644 index 000000000..728f83a35 --- /dev/null +++ b/src/common/models/quota_usage.go @@ -0,0 +1,77 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "encoding/json" + "time" +) + +// QuotaUsed a map for the quota used +type QuotaUsed map[string]int64 + +func (u QuotaUsed) String() string { + bytes, _ := json.Marshal(u) + return string(bytes) +} + +// Copy returns copied quota used +func (u QuotaUsed) Copy() QuotaUsed { + used := QuotaUsed{} + for key, value := range u { + used[key] = value + } + + return used +} + +// QuotaUsage model for quota usage +type QuotaUsage struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Reference string `orm:"column(reference)" json:"reference"` + ReferenceID string `orm:"column(reference_id)" json:"reference_id"` + Used string `orm:"column(used);type(jsonb)" json:"-"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +// TableName returns table name for orm +func (qu *QuotaUsage) TableName() string { + return "quota_usage" +} + +// GetUsed returns quota used +func (qu *QuotaUsage) GetUsed() (QuotaUsed, error) { + var used QuotaUsed + if err := json.Unmarshal([]byte(qu.Used), &used); err != nil { + return nil, err + } + + return used, nil +} + +// SetUsed set quota used +func (qu *QuotaUsage) SetUsed(used QuotaUsed) { + qu.Used = used.String() +} + +// QuotaUsageQuery query parameters for quota +type QuotaUsageQuery struct { + Reference string + ReferenceID string + ReferenceIDs []string + Pagination + Sorting +}