Add dao for quota

Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2019-07-04 11:53:26 +08:00
parent 8bb18e73d2
commit 4fedfa6580
8 changed files with 739 additions and 2 deletions

View File

@ -7,4 +7,26 @@ CREATE TABLE cve_whitelist (
expires_at bigint,
items text NOT NULL,
UNIQUE (project_id)
);
);
/* add quota table */
CREATE TABLE quota (
id SERIAL PRIMARY KEY NOT NULL,
reference VARCHAR(255) NOT NULL,
reference_id VARCHAR(255) NOT NULL,
hard JSONB NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
UNIQUE(reference, reference_id)
);
/* add quota usage table */
CREATE TABLE quota_usage (
id SERIAL PRIMARY KEY NOT NULL,
reference VARCHAR(255) NOT NULL,
reference_id VARCHAR(255) NOT NULL,
used JSONB NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
UNIQUE(reference, reference_id)
);

144
src/common/dao/quota.go Normal file
View File

@ -0,0 +1,144 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"fmt"
"strings"
"time"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/models"
)
var (
quotaOrderMap = map[string]string{
"id": "id asc",
"+id": "id asc",
"-id": "id desc",
"creation_time": "creation_time asc",
"+creation_time": "creation_time asc",
"-creation_time": "creation_time desc",
"update_time": "update_time asc",
"+update_time": "update_time asc",
"-update_time": "update_time desc",
}
)
// AddQuota add quota to the database.
func AddQuota(quota models.Quota) (int64, error) {
now := time.Now()
quota.CreationTime = now
quota.UpdateTime = now
return GetOrmer().Insert(&quota)
}
// GetQuota returns quota by id.
func GetQuota(id int64) (*models.Quota, error) {
q := models.Quota{ID: id}
err := GetOrmer().Read(&q, "ID")
if err == orm.ErrNoRows {
return nil, nil
}
return &q, err
}
// UpdateQuota update the quota.
func UpdateQuota(quota models.Quota) error {
quota.UpdateTime = time.Now()
_, err := GetOrmer().Update(&quota)
return err
}
// ListQuotas returns quotas by query.
func ListQuotas(query ...*models.QuotaQuery) ([]*models.Quota, error) {
condition, params := quotaQueryConditions(query...)
sql := fmt.Sprintf(`select * %s`, condition)
orderBy := quotaOrderBy(query...)
if orderBy != "" {
sql += ` order by ` + orderBy
}
if len(query) > 0 && query[0] != nil {
page, size := query[0].Page, query[0].Size
if size > 0 {
sql += ` limit ?`
params = append(params, size)
if page > 0 {
sql += ` offset ?`
params = append(params, size*(page-1))
}
}
}
var quotas []*models.Quota
if _, err := GetOrmer().Raw(sql, params).QueryRows(&quotas); err != nil {
return nil, err
}
return quotas, nil
}
func quotaQueryConditions(query ...*models.QuotaQuery) (string, []interface{}) {
params := []interface{}{}
sql := `from quota `
if len(query) == 0 || query[0] == nil {
return sql, params
}
sql += `where 1=1 `
q := query[0]
if q.Reference != "" {
sql += `and reference = ? `
params = append(params, q.Reference)
}
if q.ReferenceID != "" {
sql += `and reference_id = ? `
params = append(params, q.ReferenceID)
}
if len(q.ReferenceIDs) != 0 {
sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs)))
params = append(params, q.ReferenceIDs)
}
return sql, params
}
func quotaOrderBy(query ...*models.QuotaQuery) string {
orderBy := ""
if len(query) > 0 && query[0] != nil && query[0].Sort != "" {
if val, ok := quotaOrderMap[query[0].Sort]; ok {
orderBy = val
} else {
sort := query[0].Sort
order := "asc"
if sort[0] == '-' {
order = "desc"
sort = sort[1:]
}
prefix := "hard."
if strings.HasPrefix(sort, prefix) {
orderBy = fmt.Sprintf("hard->>'%s' %s", strings.TrimPrefix(sort, prefix), order)
}
}
}
return orderBy
}

View File

@ -0,0 +1,135 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"testing"
"time"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/suite"
)
var (
quotaReference = "project"
quotaUserReference = "user"
quotaHard = models.QuotaHard{"storage": 1024}
quotaHardLarger = models.QuotaHard{"storage": 2048}
)
type QuotaDaoSuite struct {
suite.Suite
}
func (suite *QuotaDaoSuite) equalHard(quota1 *models.Quota, quota2 *models.Quota) {
hard1, err := quota1.GetHard()
suite.Nil(err, "hard1 invalid")
hard2, err := quota2.GetHard()
suite.Nil(err, "hard2 invalid")
suite.Equal(hard1, hard2)
}
func (suite *QuotaDaoSuite) TearDownTest() {
ClearTable("quota")
}
func (suite *QuotaDaoSuite) TestAddQuota() {
_, err1 := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()})
suite.Nil(err1)
// Will failed for reference and reference_id should unique in db
_, err2 := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()})
suite.Error(err2)
_, err3 := AddQuota(models.Quota{Reference: quotaUserReference, ReferenceID: "1", Hard: quotaHard.String()})
suite.Nil(err3)
}
func (suite *QuotaDaoSuite) TestGetQuota() {
quota1 := models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}
id, err := AddQuota(quota1)
suite.Nil(err)
// Get the new added quota
quota2, err := GetQuota(id)
suite.Nil(err)
suite.NotNil(quota2)
// Get the quota which id is 10000 not found
quota3, err := GetQuota(10000)
suite.Nil(err)
suite.Nil(quota3)
}
func (suite *QuotaDaoSuite) TestUpdateQuota() {
quota1 := models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}
id, err := AddQuota(quota1)
suite.Nil(err)
// Get the new added quota
quota2, err := GetQuota(id)
suite.Nil(err)
suite.equalHard(&quota1, quota2)
// Update the quota
quota2.SetHard(quotaHardLarger)
time.Sleep(time.Millisecond * 10) // Ensure that UpdateTime changed
suite.Nil(UpdateQuota(*quota2))
// Get the updated quota
quota3, err := GetQuota(id)
suite.Nil(err)
suite.equalHard(quota2, quota3)
suite.NotEqual(quota2.UpdateTime, quota3.UpdateTime)
}
func (suite *QuotaDaoSuite) TestListQuotas() {
AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()})
AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "2", Hard: quotaHard.String()})
AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "3", Hard: quotaHard.String()})
AddQuota(models.Quota{Reference: quotaUserReference, ReferenceID: "1", Hard: quotaHardLarger.String()})
// List all the quotas
quotas, err := ListQuotas()
suite.Nil(err)
suite.Equal(4, len(quotas))
suite.Equal(quotaReference, quotas[0].Reference)
// List quotas filter by reference
quotas, err = ListQuotas(&models.QuotaQuery{Reference: quotaReference})
suite.Nil(err)
suite.Equal(3, len(quotas))
// List quotas filter by reference ids
quotas, err = ListQuotas(&models.QuotaQuery{Reference: quotaReference, ReferenceIDs: []string{"1", "2"}})
suite.Nil(err)
suite.Equal(2, len(quotas))
// List quotas by pagination
quotas, err = ListQuotas(&models.QuotaQuery{Pagination: models.Pagination{Size: 2}})
suite.Nil(err)
suite.Equal(2, len(quotas))
// List quotas by sorting
quotas, err = ListQuotas(&models.QuotaQuery{Sorting: models.Sorting{Sort: "-hard.storage"}})
suite.Nil(err)
suite.Equal(quotaUserReference, quotas[0].Reference)
}
func TestRunQuotaDaoSuite(t *testing.T) {
suite.Run(t, new(QuotaDaoSuite))
}

View File

@ -0,0 +1,144 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"fmt"
"strings"
"time"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/models"
)
var (
quotaUsageOrderMap = map[string]string{
"id": "id asc",
"+id": "id asc",
"-id": "id desc",
"creation_time": "creation_time asc",
"+creation_time": "creation_time asc",
"-creation_time": "creation_time desc",
"update_time": "update_time asc",
"+update_time": "update_time asc",
"-update_time": "update_time desc",
}
)
// AddQuotaUsage add quota usage to the database.
func AddQuotaUsage(quotaUsage models.QuotaUsage) (int64, error) {
now := time.Now()
quotaUsage.CreationTime = now
quotaUsage.UpdateTime = now
return GetOrmer().Insert(&quotaUsage)
}
// GetQuotaUsage returns quota usage by id.
func GetQuotaUsage(id int64) (*models.QuotaUsage, error) {
q := models.QuotaUsage{ID: id}
err := GetOrmer().Read(&q, "ID")
if err == orm.ErrNoRows {
return nil, nil
}
return &q, err
}
// UpdateQuotaUsage update the quota usage.
func UpdateQuotaUsage(quotaUsage models.QuotaUsage) error {
quotaUsage.UpdateTime = time.Now()
_, err := GetOrmer().Update(&quotaUsage)
return err
}
// ListQuotaUsages returns quota usages by query.
func ListQuotaUsages(query ...*models.QuotaUsageQuery) ([]*models.QuotaUsage, error) {
condition, params := quotaUsageQueryConditions(query...)
sql := fmt.Sprintf(`select * %s`, condition)
orderBy := quotaUsageOrderBy(query...)
if orderBy != "" {
sql += ` order by ` + orderBy
}
if len(query) > 0 && query[0] != nil {
page, size := query[0].Page, query[0].Size
if size > 0 {
sql += ` limit ?`
params = append(params, size)
if page > 0 {
sql += ` offset ?`
params = append(params, size*(page-1))
}
}
}
var quotaUsages []*models.QuotaUsage
if _, err := GetOrmer().Raw(sql, params).QueryRows(&quotaUsages); err != nil {
return nil, err
}
return quotaUsages, nil
}
func quotaUsageQueryConditions(query ...*models.QuotaUsageQuery) (string, []interface{}) {
params := []interface{}{}
sql := `from quota_usage `
if len(query) == 0 || query[0] == nil {
return sql, params
}
sql += `where 1=1 `
q := query[0]
if q.Reference != "" {
sql += `and reference = ? `
params = append(params, q.Reference)
}
if q.ReferenceID != "" {
sql += `and reference_id = ? `
params = append(params, q.ReferenceID)
}
if len(q.ReferenceIDs) != 0 {
sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs)))
params = append(params, q.ReferenceIDs)
}
return sql, params
}
func quotaUsageOrderBy(query ...*models.QuotaUsageQuery) string {
orderBy := ""
if len(query) > 0 && query[0] != nil && query[0].Sort != "" {
if val, ok := quotaUsageOrderMap[query[0].Sort]; ok {
orderBy = val
} else {
sort := query[0].Sort
order := "asc"
if sort[0] == '-' {
order = "desc"
sort = sort[1:]
}
prefix := "used."
if strings.HasPrefix(sort, prefix) {
orderBy = fmt.Sprintf("used->>'%s' %s", strings.TrimPrefix(sort, prefix), order)
}
}
}
return orderBy
}

View File

@ -0,0 +1,135 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"testing"
"time"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/suite"
)
var (
quotaUsageReference = "project"
quotaUsageUserReference = "user"
quotaUsageUsed = models.QuotaUsed{"storage": 1024}
quotaUsageUsedLarger = models.QuotaUsed{"storage": 2048}
)
type QuotaUsageDaoSuite struct {
suite.Suite
}
func (suite *QuotaUsageDaoSuite) equalUsed(usage1 *models.QuotaUsage, usage2 *models.QuotaUsage) {
used1, err := usage1.GetUsed()
suite.Nil(err, "used1 invalid")
used2, err := usage2.GetUsed()
suite.Nil(err, "used2 invalid")
suite.Equal(used1, used2)
}
func (suite *QuotaUsageDaoSuite) TearDownTest() {
ClearTable("quota_usage")
}
func (suite *QuotaUsageDaoSuite) TestAddQuotaUsage() {
_, err1 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
suite.Nil(err1)
// Will failed for reference and reference_id should unique in db
_, err2 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
suite.Error(err2)
_, err3 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageUserReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
suite.Nil(err3)
}
func (suite *QuotaUsageDaoSuite) TestGetQuotaUsage() {
quotaUsage1 := models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}
id, err := AddQuotaUsage(quotaUsage1)
suite.Nil(err)
// Get the new added quotaUsage
quotaUsage2, err := GetQuotaUsage(id)
suite.Nil(err)
suite.NotNil(quotaUsage2)
// Get the quotaUsage which id is 10000 not found
quotaUsage3, err := GetQuotaUsage(10000)
suite.Nil(err)
suite.Nil(quotaUsage3)
}
func (suite *QuotaUsageDaoSuite) TestUpdateQuotaUsage() {
quotaUsage1 := models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}
id, err := AddQuotaUsage(quotaUsage1)
suite.Nil(err)
// Get the new added quotaUsage
quotaUsage2, err := GetQuotaUsage(id)
suite.Nil(err)
suite.equalUsed(&quotaUsage1, quotaUsage2)
// Update the quotaUsage
quotaUsage2.SetUsed(quotaUsageUsedLarger)
time.Sleep(time.Millisecond * 10) // Ensure that UpdateTime changed
suite.Nil(UpdateQuotaUsage(*quotaUsage2))
// Get the updated quotaUsage
quotaUsage3, err := GetQuotaUsage(id)
suite.Nil(err)
suite.equalUsed(quotaUsage2, quotaUsage3)
suite.NotEqual(quotaUsage2.UpdateTime, quotaUsage3.UpdateTime)
}
func (suite *QuotaUsageDaoSuite) TestListQuotaUsages() {
AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "2", Used: quotaUsageUsed.String()})
AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "3", Used: quotaUsageUsed.String()})
AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageUserReference, ReferenceID: "1", Used: quotaUsageUsedLarger.String()})
// List all the quotaUsages
quotaUsages, err := ListQuotaUsages()
suite.Nil(err)
suite.Equal(4, len(quotaUsages))
suite.Equal(quotaUsageReference, quotaUsages[0].Reference)
// List quotaUsages filter by reference
quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Reference: quotaUsageReference})
suite.Nil(err)
suite.Equal(3, len(quotaUsages))
// List quotaUsages filter by reference ids
quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Reference: quotaUsageReference, ReferenceIDs: []string{"1", "2"}})
suite.Nil(err)
suite.Equal(2, len(quotaUsages))
// List quotaUsages by pagination
quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Pagination: models.Pagination{Size: 2}})
suite.Nil(err)
suite.Equal(2, len(quotaUsages))
// List quotaUsages by sorting
quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Sorting: models.Sorting{Sort: "-used.storage"}})
suite.Nil(err)
suite.Equal(quotaUsageUserReference, quotaUsages[0].Reference)
}
func TestRunQuotaUsageDaoSuite(t *testing.T) {
suite.Run(t, new(QuotaUsageDaoSuite))
}

View File

@ -37,5 +37,8 @@ func init() {
new(JobLog),
new(Robot),
new(OIDCUser),
new(CVEWhitelist))
new(CVEWhitelist),
new(Quota),
new(QuotaUsage),
)
}

View File

@ -0,0 +1,77 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"encoding/json"
"time"
)
// QuotaHard a map for the quota hard
type QuotaHard map[string]int64
func (h QuotaHard) String() string {
bytes, _ := json.Marshal(h)
return string(bytes)
}
// Copy returns copied quota hard
func (h QuotaHard) Copy() QuotaHard {
hard := QuotaHard{}
for key, value := range h {
hard[key] = value
}
return hard
}
// Quota model for quota
type Quota struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Reference string `orm:"column(reference)" json:"reference"`
ReferenceID string `orm:"column(reference_id)" json:"reference_id"`
Hard string `orm:"column(hard);type(jsonb)" json:"-"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// TableName returns table name for orm
func (q *Quota) TableName() string {
return "quota"
}
// GetHard returns quota hard
func (q *Quota) GetHard() (QuotaHard, error) {
var hard QuotaHard
if err := json.Unmarshal([]byte(q.Hard), &hard); err != nil {
return nil, err
}
return hard, nil
}
// SetHard set new quota hard
func (q *Quota) SetHard(hard QuotaHard) {
q.Hard = hard.String()
}
// QuotaQuery query parameters for quota
type QuotaQuery struct {
Reference string
ReferenceID string
ReferenceIDs []string
Pagination
Sorting
}

View File

@ -0,0 +1,77 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"encoding/json"
"time"
)
// QuotaUsed a map for the quota used
type QuotaUsed map[string]int64
func (u QuotaUsed) String() string {
bytes, _ := json.Marshal(u)
return string(bytes)
}
// Copy returns copied quota used
func (u QuotaUsed) Copy() QuotaUsed {
used := QuotaUsed{}
for key, value := range u {
used[key] = value
}
return used
}
// QuotaUsage model for quota usage
type QuotaUsage struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Reference string `orm:"column(reference)" json:"reference"`
ReferenceID string `orm:"column(reference_id)" json:"reference_id"`
Used string `orm:"column(used);type(jsonb)" json:"-"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// TableName returns table name for orm
func (qu *QuotaUsage) TableName() string {
return "quota_usage"
}
// GetUsed returns quota used
func (qu *QuotaUsage) GetUsed() (QuotaUsed, error) {
var used QuotaUsed
if err := json.Unmarshal([]byte(qu.Used), &used); err != nil {
return nil, err
}
return used, nil
}
// SetUsed set quota used
func (qu *QuotaUsage) SetUsed(used QuotaUsed) {
qu.Used = used.String()
}
// QuotaUsageQuery query parameters for quota
type QuotaUsageQuery struct {
Reference string
ReferenceID string
ReferenceIDs []string
Pagination
Sorting
}