add dao of artifact

Signed-off-by: wang yan <wangyan@vmware.com>

Add dao for quota

Signed-off-by: He Weiwei <hweiwei@vmware.com>

fix govet

Signed-off-by: wang yan <wangyan@vmware.com>

make/migrations/postgresql/0010_1.9.0_schema.up.sql

@@ -7,4 +7,66 @@ CREATE TABLE cve_whitelist (
     expires_at bigint,
     items text NOT NULL,
     UNIQUE (project_id)
-);
+);
+
+CREATE TABLE blob (
+ id SERIAL PRIMARY KEY NOT NULL,
+ /*
+    digest of config, layer, manifest
+ */
+ digest varchar(255) NOT NULL,
+ content_type varchar(255) NOT NULL,
+ size int NOT NULL,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ UNIQUE (digest)
+);
+
+CREATE TABLE artifact (
+ id SERIAL PRIMARY KEY NOT NULL,
+ project_id int NOT NULL,
+ repo varchar(255) NOT NULL,
+ tag varchar(255) NOT NULL,
+ /*
+    digest of manifest
+ */
+ digest varchar(255) NOT NULL,
+ /*
+    kind of artifact, image, chart, etc..
+ */
+ kind varchar(255) NOT NULL,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ pull_time timestamp,
+ push_time timestamp,
+ CONSTRAINT unique_artifact UNIQUE (project_id, repo, tag)
+);
+
+/* add the table for relation of artifact and blob */
+CREATE TABLE artifact_blob (
+ id SERIAL PRIMARY KEY NOT NULL,
+ digest_af varchar(255) NOT NULL,
+ digest_blob varchar(255) NOT NULL,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ CONSTRAINT unique_artifact_blob UNIQUE (digest_af, digest_blob)
+);
+
+/* add quota table */
+CREATE TABLE quota (
+ id SERIAL PRIMARY KEY NOT NULL,
+ reference VARCHAR(255) NOT NULL,
+ reference_id VARCHAR(255) NOT NULL,
+ hard JSONB NOT NULL,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP,
+ UNIQUE(reference, reference_id)
+);
+
+/* add quota usage table */
+CREATE TABLE quota_usage (
+ id SERIAL PRIMARY KEY NOT NULL,
+ reference VARCHAR(255) NOT NULL,
+ reference_id VARCHAR(255) NOT NULL,
+ used JSONB NOT NULL,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP,
+ UNIQUE(reference, reference_id)
+);

src/common/dao/artifact.go

@@ -0,0 +1,60 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"github.com/goharbor/harbor/src/common/models"
+	"strings"
+	"time"
+)
+
+// AddArtifact ...
+func AddArtifact(af *models.Artifact) (int64, error) {
+	now := time.Now()
+	af.CreationTime = now
+	id, err := GetOrmer().Insert(af)
+	if err != nil {
+		if strings.Contains(err.Error(), "duplicate key value violates unique constraint") {
+			return 0, ErrDupRows
+		}
+		return 0, err
+	}
+	return id, nil
+}
+
+// DeleteArtifact ...
+func DeleteArtifact(id int64) error {
+	_, err := GetOrmer().QueryTable(&models.Artifact{}).Filter("ID", id).Delete()
+	return err
+}
+
+// DeleteArtifactByDigest ...
+func DeleteArtifactByDigest(digest string) error {
+	_, err := GetOrmer().Raw(`delete from artifact where digest = ? `, digest).Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// DeleteByTag ...
+func DeleteByTag(projectID int, repo, tag string) error {
+	_, err := GetOrmer().Raw(`delete from artifact where project_id = ? and repo = ? and tag = ? `,
+		projectID, repo, tag).Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}

src/common/dao/artifact_blob.go

@@ -0,0 +1,107 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"fmt"
+	"github.com/astaxie/beego/orm"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/pkg/errors"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// AddArtifactNBlob ...
+func AddArtifactNBlob(afnb *models.ArtifactAndBlob) (int64, error) {
+	now := time.Now()
+	afnb.CreationTime = now
+	id, err := GetOrmer().Insert(afnb)
+	if err != nil {
+		if strings.Contains(err.Error(), "duplicate key value violates unique constraint") {
+			return 0, ErrDupRows
+		}
+		return 0, err
+	}
+	return id, nil
+}
+
+// AddArtifactNBlobs ...
+func AddArtifactNBlobs(afnbs []*models.ArtifactAndBlob) error {
+	o := orm.NewOrm()
+	err := o.Begin()
+	if err != nil {
+		return err
+	}
+
+	var errInsertMultiple error
+	total := len(afnbs)
+	successNums, err := o.InsertMulti(total, afnbs)
+	if err != nil {
+		errInsertMultiple = err
+		err := o.Rollback()
+		if err != nil {
+			log.Errorf("fail to rollback when to insert multiple artifact and blobs, %v", err)
+			errInsertMultiple = errors.Wrap(errInsertMultiple, err.Error())
+		}
+		return errInsertMultiple
+	}
+
+	// part of them cannot be inserted successfully.
+	if successNums != int64(total) {
+		errInsertMultiple = errors.New("Not all of artifact and blobs are inserted successfully")
+		err := o.Rollback()
+		if err != nil {
+			log.Errorf("fail to rollback when to insert multiple artifact and blobs, %v", err)
+			errInsertMultiple = errors.Wrap(errInsertMultiple, err.Error())
+		}
+		return errInsertMultiple
+	}
+
+	err = o.Commit()
+	if err != nil {
+		log.Errorf("fail to commit when to insert multiple artifact and blobs, %v", err)
+		return fmt.Errorf("fail to commit when to insert multiple artifact and blobs, %v", err)
+	}
+
+	return nil
+}
+
+// DeleteArtifactAndBlobByDigest ...
+func DeleteArtifactAndBlobByDigest(digest string) error {
+	_, err := GetOrmer().Raw(`delete from artifact_blob where digest_af = ? `, digest).Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// CountSizeOfArtifact ...
+func CountSizeOfArtifact(digest string) (int64, error) {
+	var res []orm.Params
+	num, err := GetOrmer().Raw(`SELECT sum(bb.size) FROM artifact_blob afnb LEFT JOIN blob bb ON afnb.digest_blob = bb.digest WHERE afnb.digest_af = ? `, digest).Values(&res)
+	if err != nil {
+		return -1, err
+	}
+	if num > 0 {
+		size, err := strconv.ParseInt(res[0]["sum"].(string), 0, 64)
+		if err != nil {
+			return -1, err
+		}
+		return size, nil
+	}
+	return -1, err
+}

src/common/dao/artifact_blob_test.go

@@ -0,0 +1,131 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/common/models"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddArtifactNBlob(t *testing.T) {
+	afnb := &models.ArtifactAndBlob{
+		DigestAF:   "vvvv",
+		DigestBlob: "aaaa",
+	}
+
+	// add
+	id, err := AddArtifactNBlob(afnb)
+	require.Nil(t, err)
+	afnb.ID = id
+	assert.Equal(t, id, int64(1))
+}
+
+func TestAddArtifactNBlobs(t *testing.T) {
+	afnb1 := &models.ArtifactAndBlob{
+		DigestAF:   "zzzz",
+		DigestBlob: "zzza",
+	}
+	afnb2 := &models.ArtifactAndBlob{
+		DigestAF:   "zzzz",
+		DigestBlob: "zzzb",
+	}
+	afnb3 := &models.ArtifactAndBlob{
+		DigestAF:   "zzzz",
+		DigestBlob: "zzzc",
+	}
+
+	var afnbs []*models.ArtifactAndBlob
+	afnbs = append(afnbs, afnb1)
+	afnbs = append(afnbs, afnb2)
+	afnbs = append(afnbs, afnb3)
+
+	// add
+	err := AddArtifactNBlobs(afnbs)
+	require.Nil(t, err)
+}
+
+func TestDeleteArtifactAndBlobByDigest(t *testing.T) {
+	afnb := &models.ArtifactAndBlob{
+		DigestAF:   "vvvv",
+		DigestBlob: "vvva",
+	}
+
+	// add
+	_, err := AddArtifactNBlob(afnb)
+	require.Nil(t, err)
+
+	// delete
+	err = DeleteArtifactAndBlobByDigest(afnb.DigestAF)
+	require.Nil(t, err)
+}
+
+func TestCountSizeOfArtifact(t *testing.T) {
+
+	afnb1 := &models.ArtifactAndBlob{
+		DigestAF:   "xxxx",
+		DigestBlob: "aaaa",
+	}
+	afnb2 := &models.ArtifactAndBlob{
+		DigestAF:   "xxxx",
+		DigestBlob: "aaab",
+	}
+	afnb3 := &models.ArtifactAndBlob{
+		DigestAF:   "xxxx",
+		DigestBlob: "aaac",
+	}
+
+	var afnbs []*models.ArtifactAndBlob
+	afnbs = append(afnbs, afnb1)
+	afnbs = append(afnbs, afnb2)
+	afnbs = append(afnbs, afnb3)
+
+	err := AddArtifactNBlobs(afnbs)
+	require.Nil(t, err)
+
+	blob1 := &models.Blob{
+		Digest:      "aaaa",
+		ContentType: "v2.blob",
+		Size:        100,
+	}
+
+	_, err = AddBlob(blob1)
+	require.Nil(t, err)
+
+	blob2 := &models.Blob{
+		Digest:      "aaab",
+		ContentType: "v2.blob",
+		Size:        200,
+	}
+
+	_, err = AddBlob(blob2)
+	require.Nil(t, err)
+
+	blob3 := &models.Blob{
+		Digest:      "aaac",
+		ContentType: "v2.blob",
+		Size:        300,
+	}
+
+	_, err = AddBlob(blob3)
+	require.Nil(t, err)
+
+	imageSize, err := CountSizeOfArtifact("xxxx")
+	require.Nil(t, err)
+	require.Equal(t, imageSize, int64(600))
+}

src/common/dao/artifact_test.go

@@ -0,0 +1,92 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/common/models"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddArtifact(t *testing.T) {
+	af := &models.Artifact{
+		PID:    1,
+		Repo:   "hello-world",
+		Tag:    "latest",
+		Digest: "1234abcd",
+		Kind:   "image",
+	}
+
+	// add
+	id, err := AddArtifact(af)
+	require.Nil(t, err)
+	af.ID = id
+	assert.Equal(t, id, int64(1))
+
+}
+
+func TestDeleteArtifact(t *testing.T) {
+	af := &models.Artifact{
+		PID:    1,
+		Repo:   "hello-world",
+		Tag:    "v1.0",
+		Digest: "1234abcd",
+		Kind:   "image",
+	}
+	// add
+	id, err := AddArtifact(af)
+	require.Nil(t, err)
+
+	// delete
+	err = DeleteArtifact(id)
+	require.Nil(t, err)
+}
+
+func TestDeleteArtifactByDigest(t *testing.T) {
+	af := &models.Artifact{
+		PID:    1,
+		Repo:   "hello-world",
+		Tag:    "v1.1",
+		Digest: "TestDeleteArtifactByDigest",
+		Kind:   "image",
+	}
+	// add
+	_, err := AddArtifact(af)
+	require.Nil(t, err)
+
+	// delete
+	err = DeleteArtifactByDigest(af.Digest)
+	require.Nil(t, err)
+}
+
+func TestDeleteArtifactByTag(t *testing.T) {
+	af := &models.Artifact{
+		PID:    1,
+		Repo:   "hello-world",
+		Tag:    "v1.2",
+		Digest: "TestDeleteArtifactByTag",
+		Kind:   "image",
+	}
+	// add
+	_, err := AddArtifact(af)
+	require.Nil(t, err)
+
+	// delete
+	err = DeleteByTag(1, "hello-world", "v1.2")
+	require.Nil(t, err)
+}

src/common/dao/blob.go

@@ -0,0 +1,50 @@
+package dao
+
+import (
+	"fmt"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"strings"
+	"time"
+)
+
+// AddBlob ...
+func AddBlob(blob *models.Blob) (int64, error) {
+	now := time.Now()
+	blob.CreationTime = now
+	id, err := GetOrmer().Insert(blob)
+	if err != nil {
+		if strings.Contains(err.Error(), "duplicate key value violates unique constraint") {
+			return 0, ErrDupRows
+		}
+		return 0, err
+	}
+	return id, nil
+}
+
+// GetBlob ...
+func GetBlob(digest string) (*models.Blob, error) {
+	o := GetOrmer()
+	qs := o.QueryTable(&models.Blob{})
+	qs = qs.Filter("Digest", digest)
+	b := []*models.Blob{}
+	_, err := qs.All(&b)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get blob for digest %s, error: %v", digest, err)
+	}
+	if len(b) == 0 {
+		log.Infof("No blob found for digest %s, returning empty.", digest)
+		return &models.Blob{}, nil
+	} else if len(b) > 1 {
+		log.Infof("Multiple blob found for digest %s", digest)
+		return &models.Blob{}, fmt.Errorf("Multiple blob found for digest %s", digest)
+	}
+	return b[0], nil
+}
+
+// DeleteBlob ...
+func DeleteBlob(digest string) error {
+	o := GetOrmer()
+	_, err := o.QueryTable("blob").Filter("digest", digest).Delete()
+	return err
+}

src/common/dao/blob_test.go

@@ -0,0 +1,65 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestAddBlob(t *testing.T) {
+	blob := &models.Blob{
+		Digest:      "1234abcd",
+		ContentType: "v2.blob",
+		Size:        1523,
+	}
+
+	// add
+	_, err := AddBlob(blob)
+	require.Nil(t, err)
+}
+
+func TestGetBlob(t *testing.T) {
+	blob := &models.Blob{
+		Digest:      "12345abcde",
+		ContentType: "v2.blob",
+		Size:        453,
+	}
+
+	// add
+	id, err := AddBlob(blob)
+	require.Nil(t, err)
+	blob.ID = id
+
+	blob2, err := GetBlob("12345abcde")
+	require.Nil(t, err)
+	assert.Equal(t, blob.Digest, blob2.Digest)
+
+}
+
+func TestDeleteBlob(t *testing.T) {
+	blob := &models.Blob{
+		Digest:      "123456abcdef",
+		ContentType: "v2.blob",
+		Size:        4543,
+	}
+	id, err := AddBlob(blob)
+	require.Nil(t, err)
+	blob.ID = id
+	err = DeleteBlob(blob.Digest)
+	require.Nil(t, err)
+}

src/common/dao/quota.go

@@ -0,0 +1,144 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/astaxie/beego/orm"
+	"github.com/goharbor/harbor/src/common/models"
+)
+
+var (
+	quotaOrderMap = map[string]string{
+		"id":             "id asc",
+		"+id":            "id asc",
+		"-id":            "id desc",
+		"creation_time":  "creation_time asc",
+		"+creation_time": "creation_time asc",
+		"-creation_time": "creation_time desc",
+		"update_time":    "update_time asc",
+		"+update_time":   "update_time asc",
+		"-update_time":   "update_time desc",
+	}
+)
+
+// AddQuota add quota to the database.
+func AddQuota(quota models.Quota) (int64, error) {
+	now := time.Now()
+	quota.CreationTime = now
+	quota.UpdateTime = now
+	return GetOrmer().Insert(&quota)
+}
+
+// GetQuota returns quota by id.
+func GetQuota(id int64) (*models.Quota, error) {
+	q := models.Quota{ID: id}
+	err := GetOrmer().Read(&q, "ID")
+	if err == orm.ErrNoRows {
+		return nil, nil
+	}
+	return &q, err
+}
+
+// UpdateQuota update the quota.
+func UpdateQuota(quota models.Quota) error {
+	quota.UpdateTime = time.Now()
+	_, err := GetOrmer().Update(&quota)
+	return err
+}
+
+// ListQuotas returns quotas by query.
+func ListQuotas(query ...*models.QuotaQuery) ([]*models.Quota, error) {
+	condition, params := quotaQueryConditions(query...)
+	sql := fmt.Sprintf(`select * %s`, condition)
+
+	orderBy := quotaOrderBy(query...)
+	if orderBy != "" {
+		sql += ` order by ` + orderBy
+	}
+
+	if len(query) > 0 && query[0] != nil {
+		page, size := query[0].Page, query[0].Size
+		if size > 0 {
+			sql += ` limit ?`
+			params = append(params, size)
+			if page > 0 {
+				sql += ` offset ?`
+				params = append(params, size*(page-1))
+			}
+		}
+	}
+
+	var quotas []*models.Quota
+	if _, err := GetOrmer().Raw(sql, params).QueryRows(&quotas); err != nil {
+		return nil, err
+	}
+
+	return quotas, nil
+}
+
+func quotaQueryConditions(query ...*models.QuotaQuery) (string, []interface{}) {
+	params := []interface{}{}
+	sql := `from quota `
+	if len(query) == 0 || query[0] == nil {
+		return sql, params
+	}
+
+	sql += `where 1=1 `
+
+	q := query[0]
+	if q.Reference != "" {
+		sql += `and reference = ? `
+		params = append(params, q.Reference)
+	}
+	if q.ReferenceID != "" {
+		sql += `and reference_id = ? `
+		params = append(params, q.ReferenceID)
+	}
+	if len(q.ReferenceIDs) != 0 {
+		sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs)))
+		params = append(params, q.ReferenceIDs)
+	}
+
+	return sql, params
+}
+
+func quotaOrderBy(query ...*models.QuotaQuery) string {
+	orderBy := ""
+
+	if len(query) > 0 && query[0] != nil && query[0].Sort != "" {
+		if val, ok := quotaOrderMap[query[0].Sort]; ok {
+			orderBy = val
+		} else {
+			sort := query[0].Sort
+
+			order := "asc"
+			if sort[0] == '-' {
+				order = "desc"
+				sort = sort[1:]
+			}
+
+			prefix := "hard."
+			if strings.HasPrefix(sort, prefix) {
+				orderBy = fmt.Sprintf("hard->>'%s' %s", strings.TrimPrefix(sort, prefix), order)
+			}
+		}
+	}
+
+	return orderBy
+}

src/common/dao/quota_test.go

@@ -0,0 +1,135 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"testing"
+	"time"
+
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/stretchr/testify/suite"
+)
+
+var (
+	quotaReference     = "project"
+	quotaUserReference = "user"
+	quotaHard          = models.QuotaHard{"storage": 1024}
+	quotaHardLarger    = models.QuotaHard{"storage": 2048}
+)
+
+type QuotaDaoSuite struct {
+	suite.Suite
+}
+
+func (suite *QuotaDaoSuite) equalHard(quota1 *models.Quota, quota2 *models.Quota) {
+	hard1, err := quota1.GetHard()
+	suite.Nil(err, "hard1 invalid")
+
+	hard2, err := quota2.GetHard()
+	suite.Nil(err, "hard2 invalid")
+
+	suite.Equal(hard1, hard2)
+}
+
+func (suite *QuotaDaoSuite) TearDownTest() {
+	ClearTable("quota")
+}
+
+func (suite *QuotaDaoSuite) TestAddQuota() {
+	_, err1 := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()})
+	suite.Nil(err1)
+
+	// Will failed for reference and reference_id should unique in db
+	_, err2 := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()})
+	suite.Error(err2)
+
+	_, err3 := AddQuota(models.Quota{Reference: quotaUserReference, ReferenceID: "1", Hard: quotaHard.String()})
+	suite.Nil(err3)
+}
+
+func (suite *QuotaDaoSuite) TestGetQuota() {
+	quota1 := models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}
+	id, err := AddQuota(quota1)
+	suite.Nil(err)
+
+	// Get the new added quota
+	quota2, err := GetQuota(id)
+	suite.Nil(err)
+	suite.NotNil(quota2)
+
+	// Get the quota which id is 10000 not found
+	quota3, err := GetQuota(10000)
+	suite.Nil(err)
+	suite.Nil(quota3)
+}
+
+func (suite *QuotaDaoSuite) TestUpdateQuota() {
+	quota1 := models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}
+	id, err := AddQuota(quota1)
+	suite.Nil(err)
+
+	// Get the new added quota
+	quota2, err := GetQuota(id)
+	suite.Nil(err)
+	suite.equalHard(&quota1, quota2)
+
+	// Update the quota
+	quota2.SetHard(quotaHardLarger)
+	time.Sleep(time.Millisecond * 10) // Ensure that UpdateTime changed
+	suite.Nil(UpdateQuota(*quota2))
+
+	// Get the updated quota
+	quota3, err := GetQuota(id)
+	suite.Nil(err)
+	suite.equalHard(quota2, quota3)
+	suite.NotEqual(quota2.UpdateTime, quota3.UpdateTime)
+}
+
+func (suite *QuotaDaoSuite) TestListQuotas() {
+	AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()})
+	AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "2", Hard: quotaHard.String()})
+	AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "3", Hard: quotaHard.String()})
+	AddQuota(models.Quota{Reference: quotaUserReference, ReferenceID: "1", Hard: quotaHardLarger.String()})
+
+	// List all the quotas
+	quotas, err := ListQuotas()
+	suite.Nil(err)
+	suite.Equal(4, len(quotas))
+	suite.Equal(quotaReference, quotas[0].Reference)
+
+	// List quotas filter by reference
+	quotas, err = ListQuotas(&models.QuotaQuery{Reference: quotaReference})
+	suite.Nil(err)
+	suite.Equal(3, len(quotas))
+
+	// List quotas filter by reference ids
+	quotas, err = ListQuotas(&models.QuotaQuery{Reference: quotaReference, ReferenceIDs: []string{"1", "2"}})
+	suite.Nil(err)
+	suite.Equal(2, len(quotas))
+
+	// List quotas by pagination
+	quotas, err = ListQuotas(&models.QuotaQuery{Pagination: models.Pagination{Size: 2}})
+	suite.Nil(err)
+	suite.Equal(2, len(quotas))
+
+	// List quotas by sorting
+	quotas, err = ListQuotas(&models.QuotaQuery{Sorting: models.Sorting{Sort: "-hard.storage"}})
+	suite.Nil(err)
+	suite.Equal(quotaUserReference, quotas[0].Reference)
+}
+
+func TestRunQuotaDaoSuite(t *testing.T) {
+	suite.Run(t, new(QuotaDaoSuite))
+}

src/common/dao/quota_usage.go

@@ -0,0 +1,144 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/astaxie/beego/orm"
+	"github.com/goharbor/harbor/src/common/models"
+)
+
+var (
+	quotaUsageOrderMap = map[string]string{
+		"id":             "id asc",
+		"+id":            "id asc",
+		"-id":            "id desc",
+		"creation_time":  "creation_time asc",
+		"+creation_time": "creation_time asc",
+		"-creation_time": "creation_time desc",
+		"update_time":    "update_time asc",
+		"+update_time":   "update_time asc",
+		"-update_time":   "update_time desc",
+	}
+)
+
+// AddQuotaUsage add quota usage to the database.
+func AddQuotaUsage(quotaUsage models.QuotaUsage) (int64, error) {
+	now := time.Now()
+	quotaUsage.CreationTime = now
+	quotaUsage.UpdateTime = now
+	return GetOrmer().Insert(&quotaUsage)
+}
+
+// GetQuotaUsage returns quota usage by id.
+func GetQuotaUsage(id int64) (*models.QuotaUsage, error) {
+	q := models.QuotaUsage{ID: id}
+	err := GetOrmer().Read(&q, "ID")
+	if err == orm.ErrNoRows {
+		return nil, nil
+	}
+	return &q, err
+}
+
+// UpdateQuotaUsage update the quota usage.
+func UpdateQuotaUsage(quotaUsage models.QuotaUsage) error {
+	quotaUsage.UpdateTime = time.Now()
+	_, err := GetOrmer().Update(&quotaUsage)
+	return err
+}
+
+// ListQuotaUsages returns quota usages by query.
+func ListQuotaUsages(query ...*models.QuotaUsageQuery) ([]*models.QuotaUsage, error) {
+	condition, params := quotaUsageQueryConditions(query...)
+	sql := fmt.Sprintf(`select * %s`, condition)
+
+	orderBy := quotaUsageOrderBy(query...)
+	if orderBy != "" {
+		sql += ` order by ` + orderBy
+	}
+
+	if len(query) > 0 && query[0] != nil {
+		page, size := query[0].Page, query[0].Size
+		if size > 0 {
+			sql += ` limit ?`
+			params = append(params, size)
+			if page > 0 {
+				sql += ` offset ?`
+				params = append(params, size*(page-1))
+			}
+		}
+	}
+
+	var quotaUsages []*models.QuotaUsage
+	if _, err := GetOrmer().Raw(sql, params).QueryRows(&quotaUsages); err != nil {
+		return nil, err
+	}
+
+	return quotaUsages, nil
+}
+
+func quotaUsageQueryConditions(query ...*models.QuotaUsageQuery) (string, []interface{}) {
+	params := []interface{}{}
+	sql := `from quota_usage `
+	if len(query) == 0 || query[0] == nil {
+		return sql, params
+	}
+
+	sql += `where 1=1 `
+
+	q := query[0]
+	if q.Reference != "" {
+		sql += `and reference = ? `
+		params = append(params, q.Reference)
+	}
+	if q.ReferenceID != "" {
+		sql += `and reference_id = ? `
+		params = append(params, q.ReferenceID)
+	}
+	if len(q.ReferenceIDs) != 0 {
+		sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs)))
+		params = append(params, q.ReferenceIDs)
+	}
+
+	return sql, params
+}
+
+func quotaUsageOrderBy(query ...*models.QuotaUsageQuery) string {
+	orderBy := ""
+
+	if len(query) > 0 && query[0] != nil && query[0].Sort != "" {
+		if val, ok := quotaUsageOrderMap[query[0].Sort]; ok {
+			orderBy = val
+		} else {
+			sort := query[0].Sort
+
+			order := "asc"
+			if sort[0] == '-' {
+				order = "desc"
+				sort = sort[1:]
+			}
+
+			prefix := "used."
+			if strings.HasPrefix(sort, prefix) {
+				orderBy = fmt.Sprintf("used->>'%s' %s", strings.TrimPrefix(sort, prefix), order)
+			}
+		}
+	}
+
+	return orderBy
+}

src/common/dao/quota_usage_test.go

@@ -0,0 +1,135 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"testing"
+	"time"
+
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/stretchr/testify/suite"
+)
+
+var (
+	quotaUsageReference     = "project"
+	quotaUsageUserReference = "user"
+	quotaUsageUsed          = models.QuotaUsed{"storage": 1024}
+	quotaUsageUsedLarger    = models.QuotaUsed{"storage": 2048}
+)
+
+type QuotaUsageDaoSuite struct {
+	suite.Suite
+}
+
+func (suite *QuotaUsageDaoSuite) equalUsed(usage1 *models.QuotaUsage, usage2 *models.QuotaUsage) {
+	used1, err := usage1.GetUsed()
+	suite.Nil(err, "used1 invalid")
+
+	used2, err := usage2.GetUsed()
+	suite.Nil(err, "used2 invalid")
+
+	suite.Equal(used1, used2)
+}
+
+func (suite *QuotaUsageDaoSuite) TearDownTest() {
+	ClearTable("quota_usage")
+}
+
+func (suite *QuotaUsageDaoSuite) TestAddQuotaUsage() {
+	_, err1 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
+	suite.Nil(err1)
+
+	// Will failed for reference and reference_id should unique in db
+	_, err2 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
+	suite.Error(err2)
+
+	_, err3 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageUserReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
+	suite.Nil(err3)
+}
+
+func (suite *QuotaUsageDaoSuite) TestGetQuotaUsage() {
+	quotaUsage1 := models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}
+	id, err := AddQuotaUsage(quotaUsage1)
+	suite.Nil(err)
+
+	// Get the new added quotaUsage
+	quotaUsage2, err := GetQuotaUsage(id)
+	suite.Nil(err)
+	suite.NotNil(quotaUsage2)
+
+	// Get the quotaUsage which id is 10000 not found
+	quotaUsage3, err := GetQuotaUsage(10000)
+	suite.Nil(err)
+	suite.Nil(quotaUsage3)
+}
+
+func (suite *QuotaUsageDaoSuite) TestUpdateQuotaUsage() {
+	quotaUsage1 := models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}
+	id, err := AddQuotaUsage(quotaUsage1)
+	suite.Nil(err)
+
+	// Get the new added quotaUsage
+	quotaUsage2, err := GetQuotaUsage(id)
+	suite.Nil(err)
+	suite.equalUsed(&quotaUsage1, quotaUsage2)
+
+	// Update the quotaUsage
+	quotaUsage2.SetUsed(quotaUsageUsedLarger)
+	time.Sleep(time.Millisecond * 10) // Ensure that UpdateTime changed
+	suite.Nil(UpdateQuotaUsage(*quotaUsage2))
+
+	// Get the updated quotaUsage
+	quotaUsage3, err := GetQuotaUsage(id)
+	suite.Nil(err)
+	suite.equalUsed(quotaUsage2, quotaUsage3)
+	suite.NotEqual(quotaUsage2.UpdateTime, quotaUsage3.UpdateTime)
+}
+
+func (suite *QuotaUsageDaoSuite) TestListQuotaUsages() {
+	AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
+	AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "2", Used: quotaUsageUsed.String()})
+	AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "3", Used: quotaUsageUsed.String()})
+	AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageUserReference, ReferenceID: "1", Used: quotaUsageUsedLarger.String()})
+
+	// List all the quotaUsages
+	quotaUsages, err := ListQuotaUsages()
+	suite.Nil(err)
+	suite.Equal(4, len(quotaUsages))
+	suite.Equal(quotaUsageReference, quotaUsages[0].Reference)
+
+	// List quotaUsages filter by reference
+	quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Reference: quotaUsageReference})
+	suite.Nil(err)
+	suite.Equal(3, len(quotaUsages))
+
+	// List quotaUsages filter by reference ids
+	quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Reference: quotaUsageReference, ReferenceIDs: []string{"1", "2"}})
+	suite.Nil(err)
+	suite.Equal(2, len(quotaUsages))
+
+	// List quotaUsages by pagination
+	quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Pagination: models.Pagination{Size: 2}})
+	suite.Nil(err)
+	suite.Equal(2, len(quotaUsages))
+
+	// List quotaUsages by sorting
+	quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Sorting: models.Sorting{Sort: "-used.storage"}})
+	suite.Nil(err)
+	suite.Equal(quotaUsageUserReference, quotaUsages[0].Reference)
+}
+
+func TestRunQuotaUsageDaoSuite(t *testing.T) {
+	suite.Run(t, new(QuotaUsageDaoSuite))
+}

src/common/models/artifact.go

@@ -0,0 +1,23 @@
+package models
+
+import (
+	"time"
+)
+
+// Artifact holds the details of a artifact.
+type Artifact struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	PID          int64     `orm:"column(project_id)" json:"project_id"`
+	Repo         string    `orm:"column(repo)" json:"repo"`
+	Tag          string    `orm:"column(tag)" json:"tag"`
+	Digest       string    `orm:"column(digest)" json:"digest"`
+	Kind         string    `orm:"column(kind)" json:"kind"`
+	PushTime     time.Time `orm:"column(push_time)" json:"push_time"`
+	PullTime     time.Time `orm:"column(pull_time)" json:"pull_time"`
+	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+}
+
+// TableName ...
+func (af *Artifact) TableName() string {
+	return "artifact"
+}

src/common/models/artifact_blob.go

@@ -0,0 +1,18 @@
+package models
+
+import (
+	"time"
+)
+
+// ArtifactAndBlob holds the relationship between manifest and blob.
+type ArtifactAndBlob struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	DigestAF     string    `orm:"column(digest_af)" json:"digest_af"`
+	DigestBlob   string    `orm:"column(digest_blob)" json:"digest_blob"`
+	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+}
+
+// TableName ...
+func (afb *ArtifactAndBlob) TableName() string {
+	return "artifact_blob"
+}

src/common/models/base.go

@@ -37,5 +37,11 @@ func init() {
 		new(JobLog),
 		new(Robot),
 		new(OIDCUser),
-		new(CVEWhitelist))
+		new(Blob),
+		new(Artifact),
+		new(ArtifactAndBlob),
+		new(CVEWhitelist),
+		new(Quota),
+		new(QuotaUsage),
+	)
 }

src/common/models/blob.go

@@ -0,0 +1,19 @@
+package models
+
+import (
+	"time"
+)
+
+// Blob holds the details of a blob.
+type Blob struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	Digest       string    `orm:"column(digest)" json:"digest"`
+	ContentType  string    `orm:"column(content_type)" json:"content_type"`
+	Size         int64     `orm:"column(size)" json:"size"`
+	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+}
+
+// TableName ...
+func (b *Blob) TableName() string {
+	return "blob"
+}

src/common/models/quota.go

@@ -0,0 +1,77 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// QuotaHard a map for the quota hard
+type QuotaHard map[string]int64
+
+func (h QuotaHard) String() string {
+	bytes, _ := json.Marshal(h)
+	return string(bytes)
+}
+
+// Copy returns copied quota hard
+func (h QuotaHard) Copy() QuotaHard {
+	hard := QuotaHard{}
+	for key, value := range h {
+		hard[key] = value
+	}
+
+	return hard
+}
+
+// Quota model for quota
+type Quota struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	Reference    string    `orm:"column(reference)" json:"reference"`
+	ReferenceID  string    `orm:"column(reference_id)" json:"reference_id"`
+	Hard         string    `orm:"column(hard);type(jsonb)" json:"-"`
+	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+	UpdateTime   time.Time `orm:"column(update_time);auto_now" json:"update_time"`
+}
+
+// TableName returns table name for orm
+func (q *Quota) TableName() string {
+	return "quota"
+}
+
+// GetHard returns quota hard
+func (q *Quota) GetHard() (QuotaHard, error) {
+	var hard QuotaHard
+	if err := json.Unmarshal([]byte(q.Hard), &hard); err != nil {
+		return nil, err
+	}
+
+	return hard, nil
+}
+
+// SetHard set new quota hard
+func (q *Quota) SetHard(hard QuotaHard) {
+	q.Hard = hard.String()
+}
+
+// QuotaQuery query parameters for quota
+type QuotaQuery struct {
+	Reference    string
+	ReferenceID  string
+	ReferenceIDs []string
+	Pagination
+	Sorting
+}

src/common/models/quota_usage.go

@@ -0,0 +1,77 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// QuotaUsed a map for the quota used
+type QuotaUsed map[string]int64
+
+func (u QuotaUsed) String() string {
+	bytes, _ := json.Marshal(u)
+	return string(bytes)
+}
+
+// Copy returns copied quota used
+func (u QuotaUsed) Copy() QuotaUsed {
+	used := QuotaUsed{}
+	for key, value := range u {
+		used[key] = value
+	}
+
+	return used
+}
+
+// QuotaUsage model for quota usage
+type QuotaUsage struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	Reference    string    `orm:"column(reference)" json:"reference"`
+	ReferenceID  string    `orm:"column(reference_id)" json:"reference_id"`
+	Used         string    `orm:"column(used);type(jsonb)" json:"-"`
+	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+	UpdateTime   time.Time `orm:"column(update_time);auto_now" json:"update_time"`
+}
+
+// TableName returns table name for orm
+func (qu *QuotaUsage) TableName() string {
+	return "quota_usage"
+}
+
+// GetUsed returns quota used
+func (qu *QuotaUsage) GetUsed() (QuotaUsed, error) {
+	var used QuotaUsed
+	if err := json.Unmarshal([]byte(qu.Used), &used); err != nil {
+		return nil, err
+	}
+
+	return used, nil
+}
+
+// SetUsed set quota used
+func (qu *QuotaUsage) SetUsed(used QuotaUsed) {
+	qu.Used = used.String()
+}
+
+// QuotaUsageQuery query parameters for quota
+type QuotaUsageQuery struct {
+	Reference    string
+	ReferenceID  string
+	ReferenceIDs []string
+	Pagination
+	Sorting
+}