Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-23 17:17:46 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #8633 from heww/quota-switch

Browse Source 
feat(quota,middleware): enable or disable quota per project by config
This commit is contained in:
Wang Yan 2019-08-12 19:36:38 +08:00 committed by GitHub
commit 2a3192b5c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 170 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/swagger.yaml
View File

@ -4841,6 +4841,9 @@ definitions:
project_creation_restriction:
type: string
description: This attribute restricts what users have the permission to create project. It can be "everyone" or "adminonly".
quota_per_project_enable:
type: boolean
description: This attribute indicates whether quota per project enabled in harbor
read_only:
type: boolean
description: '''docker push'' is prohibited by Harbor if you set it to true. '
@ -4938,6 +4941,9 @@ definitions:
project_creation_restriction:
$ref: '#/definitions/StringConfigItem'
description: This attribute restricts what users have the permission to create project. It can be "everyone" or "adminonly".
quota_per_project_enable:
$ref: '#/definitions/BoolConfigItem'
description: This attribute indicates whether quota per project enabled in harbor
read_only:
$ref: '#/definitions/BoolConfigItem'
description: '''docker push'' is prohibited by Harbor if you set it to true. '

1
src/common/config/metadata/metadatalist.go
View File

@ -151,6 +151,7 @@ var (
{Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true},
{Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true},
{Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true},
{Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true},
{Name: common.StoragePerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "STORAGE_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true},
}

6
src/common/const.go
View File

@ -146,7 +146,9 @@ const (
// Global notification enable configuration
NotificationEnable = "notification_enable"
// Quota setting items for project
CountPerProject = "count_per_project"
StoragePerProject = "storage_per_project"
QuotaPerProjectEnable = "quota_per_project_enable"
CountPerProject = "count_per_project"
StoragePerProject = "storage_per_project"
)

56
src/core/api/project.go
View File

@ -139,23 +139,26 @@ func (p *ProjectAPI) Post() {
return
}
setting, err := config.QuotaSetting()
if err != nil {
log.Errorf("failed to get quota setting: %v", err)
p.SendInternalServerError(fmt.Errorf("failed to get quota setting: %v", err))
return
}
var hardLimits types.ResourceList
if config.QuotaPerProjectEnable() {
setting, err := config.QuotaSetting()
if err != nil {
log.Errorf("failed to get quota setting: %v", err)
p.SendInternalServerError(fmt.Errorf("failed to get quota setting: %v", err))
return
}
if !p.SecurityCtx.IsSysAdmin() {
pro.CountLimit = &setting.CountPerProject
pro.StorageLimit = &setting.StoragePerProject
}
if !p.SecurityCtx.IsSysAdmin() {
pro.CountLimit = &setting.CountPerProject
pro.StorageLimit = &setting.StoragePerProject
}
hardLimits, err := projectQuotaHardLimits(pro, setting)
if err != nil {
log.Errorf("Invalid project request, error: %v", err)
p.SendBadRequestError(fmt.Errorf("invalid request: %v", err))
return
hardLimits, err = projectQuotaHardLimits(pro, setting)
if err != nil {
log.Errorf("Invalid project request, error: %v", err)
p.SendBadRequestError(fmt.Errorf("invalid request: %v", err))
return
}
}
exist, err := p.ProjectMgr.Exists(pro.Name)
@ -212,14 +215,16 @@ func (p *ProjectAPI) Post() {
return
}
quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10))
if err != nil {
p.SendInternalServerError(fmt.Errorf("failed to get quota manager: %v", err))
return
}
if _, err := quotaMgr.NewQuota(hardLimits); err != nil {
p.SendInternalServerError(fmt.Errorf("failed to create quota for project: %v", err))
return
if config.QuotaPerProjectEnable() {
quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10))
if err != nil {
p.SendInternalServerError(fmt.Errorf("failed to get quota manager: %v", err))
return
}
if _, err := quotaMgr.NewQuota(hardLimits); err != nil {
p.SendInternalServerError(fmt.Errorf("failed to create quota for project: %v", err))
return
}
}
go func() {
@ -653,6 +658,11 @@ func projectQuotaHardLimits(req *models.ProjectRequest, setting *models.QuotaSet
}
func getProjectQuotaSummary(projectID int64, summary *models.ProjectSummary) {
if !config.QuotaPerProjectEnable() {
log.Debug("Quota per project disabled")
return
}
quotas, err := dao.ListQuotas(&models.QuotaQuery{Reference: "project", ReferenceID: strconv.FormatInt(projectID, 10)})
if err != nil {
log.Debugf("failed to get quota for project: %d", projectID)

5
src/core/config/config.go
View File

@ -520,6 +520,11 @@ func NotificationEnable() bool {
return cfgMgr.Get(common.NotificationEnable).GetBool()
}
// QuotaPerProjectEnable returns a bool to indicates if quota per project enabled in harbor
func QuotaPerProjectEnable() bool {
return cfgMgr.Get(common.QuotaPerProjectEnable).GetBool()
}
// QuotaSetting returns the setting of quota.
func QuotaSetting() (*models.QuotaSetting, error) {
if err := cfgMgr.Load(); err != nil {

3
src/core/middlewares/chart/builder.go
View File

@ -21,6 +21,7 @@ import (
"strconv"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/interceptor"
"github.com/goharbor/harbor/src/core/middlewares/interceptor/quota"
"github.com/goharbor/harbor/src/core/middlewares/util"
@ -69,6 +70,7 @@ func (*chartVersionDeletionBuilder) Build(req *http.Request) (interceptor.Interc
}
opts := []quota.Option{
quota.EnforceResources(config.QuotaPerProjectEnable()),
quota.WithManager("project", strconv.FormatInt(project.ProjectID, 10)),
quota.WithAction(quota.SubtractAction),
quota.StatusCode(http.StatusOK),
@ -117,6 +119,7 @@ func (*chartVersionCreationBuilder) Build(req *http.Request) (interceptor.Interc
*req = *req.WithContext(util.NewChartVersionInfoContext(req.Context(), info))
opts := []quota.Option{
quota.EnforceResources(config.QuotaPerProjectEnable()),
quota.WithManager("project", strconv.FormatInt(project.ProjectID, 10)),
quota.WithAction(quota.AddAction),
quota.StatusCode(http.StatusCreated),

3
src/core/middlewares/countquota/builder.go
View File

@ -20,6 +20,7 @@ import (
"strconv"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/interceptor"
"github.com/goharbor/harbor/src/core/middlewares/interceptor/quota"
"github.com/goharbor/harbor/src/core/middlewares/util"
@ -52,6 +53,7 @@ func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Intercepto
}
opts := []quota.Option{
quota.EnforceResources(config.QuotaPerProjectEnable()),
quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)),
quota.WithAction(quota.SubtractAction),
quota.StatusCode(http.StatusAccepted),
@ -85,6 +87,7 @@ func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Intercepto
}
opts := []quota.Option{
quota.EnforceResources(config.QuotaPerProjectEnable()),
quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)),
quota.WithAction(quota.AddAction),
quota.StatusCode(http.StatusCreated),

2
src/core/middlewares/countquota/handler_test.go
View File

@ -26,6 +26,7 @@ import (
"github.com/docker/distribution"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"github.com/goharbor/harbor/src/pkg/types"
"github.com/opencontainers/go-digest"
@ -290,6 +291,7 @@ func (suite *HandlerSuite) TestDeleteManifestInMultiProjects() {
}
func TestMain(m *testing.M) {
config.Init()
dao.PrepareTestForPostgresSQL()
if result := m.Run(); result != 0 {

22
src/core/middlewares/interceptor/quota/options.go
View File

@ -36,6 +36,8 @@ const (
// Options ...
type Options struct {
enforceResources *bool
Action Action
Manager *quota.Manager
MutexKeys []string
@ -48,6 +50,15 @@ type Options struct {
OnFinally func(http.ResponseWriter, *http.Request) error
}
// EnforceResources ...
func (opts *Options) EnforceResources() bool {
return opts.enforceResources != nil && *opts.enforceResources
}
func boolPtr(v bool) *bool {
return &v
}
func newOptions(opt ...Option) Options {
opts := Options{}
@ -63,9 +74,20 @@ func newOptions(opt ...Option) Options {
opts.StatusCode = http.StatusOK
}
if opts.enforceResources == nil {
opts.enforceResources = boolPtr(true)
}
return opts
}
// EnforceResources sets the interceptor enforceResources
func EnforceResources(enforceResources bool) Option {
return func(o *Options) {
o.enforceResources = boolPtr(enforceResources)
}
}
// WithAction sets the interceptor action
func WithAction(a Action) Option {
return func(o *Options) {

68
src/core/middlewares/interceptor/quota/quota.go
View File

@ -49,28 +49,19 @@ func (qi *quotaInterceptor) HandleRequest(req *http.Request) (err error) {
}
}()
opts := qi.opts
for _, key := range opts.MutexKeys {
m, err := redis.RequireLock(key)
if err != nil {
return err
}
qi.mutexes = append(qi.mutexes, m)
err = qi.requireMutexes()
if err != nil {
return
}
resources := opts.Resources
if len(resources) == 0 && opts.OnResources != nil {
resources, err = opts.OnResources(req)
if err != nil {
return fmt.Errorf("failed to compute the resources for quota, error: %v", err)
}
err = qi.computeResources(req)
if err != nil {
return
}
qi.resources = resources
err = qi.reserve()
if err != nil {
log.Errorf("Failed to %s resources, error: %v", opts.Action, err)
log.Errorf("Failed to %s resources, error: %v", qi.opts.Action, err)
}
return
@ -113,6 +104,23 @@ func (qi *quotaInterceptor) HandleResponse(w http.ResponseWriter, req *http.Requ
}
}
func (qi *quotaInterceptor) requireMutexes() error {
if !qi.opts.EnforceResources() {
// Do nothing for locks when quota interceptor not enforce resources
return nil
}
for _, key := range qi.opts.MutexKeys {
m, err := redis.RequireLock(key)
if err != nil {
return err
}
qi.mutexes = append(qi.mutexes, m)
}
return nil
}
func (qi *quotaInterceptor) freeMutexes() {
for i := len(qi.mutexes) - 1; i >= 0; i-- {
if err := redis.FreeLock(qi.mutexes[i]); err != nil {
@ -121,7 +129,30 @@ func (qi *quotaInterceptor) freeMutexes() {
}
}
func (qi *quotaInterceptor) computeResources(req *http.Request) error {
if !qi.opts.EnforceResources() {
// Do nothing in compute resources when quota interceptor not enforce resources
return nil
}
if len(qi.opts.Resources) == 0 && qi.opts.OnResources != nil {
resources, err := qi.opts.OnResources(req)
if err != nil {
return fmt.Errorf("failed to compute the resources for quota, error: %v", err)
}
qi.resources = resources
}
return nil
}
func (qi *quotaInterceptor) reserve() error {
if !qi.opts.EnforceResources() {
// Do nothing in reserve resources when quota interceptor not enforce resources
return nil
}
if len(qi.resources) == 0 {
return nil
}
@ -137,6 +168,11 @@ func (qi *quotaInterceptor) reserve() error {
}
func (qi *quotaInterceptor) unreserve() error {
if !qi.opts.EnforceResources() {
// Do nothing in unreserve resources when quota interceptor not enforce resources
return nil
}
if len(qi.resources) == 0 {
return nil
}

4
src/core/middlewares/sizequota/builder.go
View File

@ -22,6 +22,7 @@ import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/interceptor"
"github.com/goharbor/harbor/src/core/middlewares/interceptor/quota"
"github.com/goharbor/harbor/src/core/middlewares/util"
@ -89,6 +90,7 @@ func (*blobStorageQuotaBuilder) Build(req *http.Request) (interceptor.Intercepto
*req = *(req.WithContext(util.NewBlobInfoContext(req.Context(), info)))
opts := []quota.Option{
quota.EnforceResources(config.QuotaPerProjectEnable()),
quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)),
quota.WithAction(quota.AddAction),
quota.StatusCode(http.StatusCreated), // NOTICE: mount blob and blob upload complete both return 201 when success
@ -119,6 +121,7 @@ func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Intercepto
*req = *req.WithContext(util.NewManifestInfoContext(req.Context(), info))
opts := []quota.Option{
quota.EnforceResources(config.QuotaPerProjectEnable()),
quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)),
quota.WithAction(quota.AddAction),
quota.StatusCode(http.StatusCreated),
@ -181,6 +184,7 @@ func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Intercepto
}
opts := []quota.Option{
quota.EnforceResources(config.QuotaPerProjectEnable()),
quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)),
quota.WithAction(quota.SubtractAction),
quota.StatusCode(http.StatusAccepted),

35
src/core/middlewares/sizequota/handler_test.go
View File

@ -30,8 +30,10 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/countquota"
"github.com/goharbor/harbor/src/core/middlewares/util"
"github.com/goharbor/harbor/src/pkg/types"
@ -662,7 +664,40 @@ func (suite *HandlerSuite) TestDeleteImageRace() {
})
}
func (suite *HandlerSuite) TestDisableProjectQuota() {
withProject(func(projectID int64, projectName string) {
manifest := makeManifest(1, []int64{2, 3, 4, 5})
pushImage(projectName, "photon", "latest", manifest)
quotas, err := dao.ListQuotas(&models.QuotaQuery{
Reference: "project",
ReferenceID: strconv.FormatInt(projectID, 10),
})
suite.Nil(err)
suite.Len(quotas, 1)
})
withProject(func(projectID int64, projectName string) {
cfg := config.GetCfgManager()
cfg.Set(common.QuotaPerProjectEnable, false)
defer cfg.Set(common.QuotaPerProjectEnable, true)
manifest := makeManifest(1, []int64{2, 3, 4, 5})
pushImage(projectName, "photon", "latest", manifest)
quotas, err := dao.ListQuotas(&models.QuotaQuery{
Reference: "project",
ReferenceID: strconv.FormatInt(projectID, 10),
})
suite.Nil(err)
suite.Len(quotas, 0)
})
}
func TestMain(m *testing.M) {
config.Init()
dao.PrepareTestForPostgresSQL()
if result := m.Run(); result != 0 {