diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 01071390b..30af9717a 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -4841,6 +4841,9 @@ definitions: project_creation_restriction: type: string description: This attribute restricts what users have the permission to create project. It can be "everyone" or "adminonly". + quota_per_project_enable: + type: boolean + description: This attribute indicates whether quota per project enabled in harbor read_only: type: boolean description: '''docker push'' is prohibited by Harbor if you set it to true. ' @@ -4938,6 +4941,9 @@ definitions: project_creation_restriction: $ref: '#/definitions/StringConfigItem' description: This attribute restricts what users have the permission to create project. It can be "everyone" or "adminonly". + quota_per_project_enable: + $ref: '#/definitions/BoolConfigItem' + description: This attribute indicates whether quota per project enabled in harbor read_only: $ref: '#/definitions/BoolConfigItem' description: '''docker push'' is prohibited by Harbor if you set it to true. ' diff --git a/src/common/config/metadata/metadatalist.go b/src/common/config/metadata/metadatalist.go index 3aa42f619..e5f692eda 100644 --- a/src/common/config/metadata/metadatalist.go +++ b/src/common/config/metadata/metadatalist.go @@ -151,6 +151,7 @@ var ( {Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true}, {Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, + {Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, {Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, {Name: common.StoragePerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "STORAGE_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, } diff --git a/src/common/const.go b/src/common/const.go index f2778a48e..b7fbc6210 100755 --- a/src/common/const.go +++ b/src/common/const.go @@ -146,7 +146,9 @@ const ( // Global notification enable configuration NotificationEnable = "notification_enable" + // Quota setting items for project - CountPerProject = "count_per_project" - StoragePerProject = "storage_per_project" + QuotaPerProjectEnable = "quota_per_project_enable" + CountPerProject = "count_per_project" + StoragePerProject = "storage_per_project" ) diff --git a/src/core/api/project.go b/src/core/api/project.go index 4a71dd316..1c98242ca 100644 --- a/src/core/api/project.go +++ b/src/core/api/project.go @@ -139,23 +139,26 @@ func (p *ProjectAPI) Post() { return } - setting, err := config.QuotaSetting() - if err != nil { - log.Errorf("failed to get quota setting: %v", err) - p.SendInternalServerError(fmt.Errorf("failed to get quota setting: %v", err)) - return - } + var hardLimits types.ResourceList + if config.QuotaPerProjectEnable() { + setting, err := config.QuotaSetting() + if err != nil { + log.Errorf("failed to get quota setting: %v", err) + p.SendInternalServerError(fmt.Errorf("failed to get quota setting: %v", err)) + return + } - if !p.SecurityCtx.IsSysAdmin() { - pro.CountLimit = &setting.CountPerProject - pro.StorageLimit = &setting.StoragePerProject - } + if !p.SecurityCtx.IsSysAdmin() { + pro.CountLimit = &setting.CountPerProject + pro.StorageLimit = &setting.StoragePerProject + } - hardLimits, err := projectQuotaHardLimits(pro, setting) - if err != nil { - log.Errorf("Invalid project request, error: %v", err) - p.SendBadRequestError(fmt.Errorf("invalid request: %v", err)) - return + hardLimits, err = projectQuotaHardLimits(pro, setting) + if err != nil { + log.Errorf("Invalid project request, error: %v", err) + p.SendBadRequestError(fmt.Errorf("invalid request: %v", err)) + return + } } exist, err := p.ProjectMgr.Exists(pro.Name) @@ -212,14 +215,16 @@ func (p *ProjectAPI) Post() { return } - quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10)) - if err != nil { - p.SendInternalServerError(fmt.Errorf("failed to get quota manager: %v", err)) - return - } - if _, err := quotaMgr.NewQuota(hardLimits); err != nil { - p.SendInternalServerError(fmt.Errorf("failed to create quota for project: %v", err)) - return + if config.QuotaPerProjectEnable() { + quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10)) + if err != nil { + p.SendInternalServerError(fmt.Errorf("failed to get quota manager: %v", err)) + return + } + if _, err := quotaMgr.NewQuota(hardLimits); err != nil { + p.SendInternalServerError(fmt.Errorf("failed to create quota for project: %v", err)) + return + } } go func() { @@ -653,6 +658,11 @@ func projectQuotaHardLimits(req *models.ProjectRequest, setting *models.QuotaSet } func getProjectQuotaSummary(projectID int64, summary *models.ProjectSummary) { + if !config.QuotaPerProjectEnable() { + log.Debug("Quota per project disabled") + return + } + quotas, err := dao.ListQuotas(&models.QuotaQuery{Reference: "project", ReferenceID: strconv.FormatInt(projectID, 10)}) if err != nil { log.Debugf("failed to get quota for project: %d", projectID) diff --git a/src/core/config/config.go b/src/core/config/config.go index 57c02bad1..d43b18f07 100755 --- a/src/core/config/config.go +++ b/src/core/config/config.go @@ -520,6 +520,11 @@ func NotificationEnable() bool { return cfgMgr.Get(common.NotificationEnable).GetBool() } +// QuotaPerProjectEnable returns a bool to indicates if quota per project enabled in harbor +func QuotaPerProjectEnable() bool { + return cfgMgr.Get(common.QuotaPerProjectEnable).GetBool() +} + // QuotaSetting returns the setting of quota. func QuotaSetting() (*models.QuotaSetting, error) { if err := cfgMgr.Load(); err != nil { diff --git a/src/core/middlewares/chart/builder.go b/src/core/middlewares/chart/builder.go index 669509ff4..19827e2d3 100644 --- a/src/core/middlewares/chart/builder.go +++ b/src/core/middlewares/chart/builder.go @@ -21,6 +21,7 @@ import ( "strconv" "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares/interceptor" "github.com/goharbor/harbor/src/core/middlewares/interceptor/quota" "github.com/goharbor/harbor/src/core/middlewares/util" @@ -69,6 +70,7 @@ func (*chartVersionDeletionBuilder) Build(req *http.Request) (interceptor.Interc } opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(project.ProjectID, 10)), quota.WithAction(quota.SubtractAction), quota.StatusCode(http.StatusOK), @@ -117,6 +119,7 @@ func (*chartVersionCreationBuilder) Build(req *http.Request) (interceptor.Interc *req = *req.WithContext(util.NewChartVersionInfoContext(req.Context(), info)) opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(project.ProjectID, 10)), quota.WithAction(quota.AddAction), quota.StatusCode(http.StatusCreated), diff --git a/src/core/middlewares/countquota/builder.go b/src/core/middlewares/countquota/builder.go index 5de9a2735..089c4a5d6 100644 --- a/src/core/middlewares/countquota/builder.go +++ b/src/core/middlewares/countquota/builder.go @@ -20,6 +20,7 @@ import ( "strconv" "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares/interceptor" "github.com/goharbor/harbor/src/core/middlewares/interceptor/quota" "github.com/goharbor/harbor/src/core/middlewares/util" @@ -52,6 +53,7 @@ func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Intercepto } opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)), quota.WithAction(quota.SubtractAction), quota.StatusCode(http.StatusAccepted), @@ -85,6 +87,7 @@ func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Intercepto } opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)), quota.WithAction(quota.AddAction), quota.StatusCode(http.StatusCreated), diff --git a/src/core/middlewares/countquota/handler_test.go b/src/core/middlewares/countquota/handler_test.go index a25166734..a2ebb5a69 100644 --- a/src/core/middlewares/countquota/handler_test.go +++ b/src/core/middlewares/countquota/handler_test.go @@ -26,6 +26,7 @@ import ( "github.com/docker/distribution" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares/util" "github.com/goharbor/harbor/src/pkg/types" "github.com/opencontainers/go-digest" @@ -290,6 +291,7 @@ func (suite *HandlerSuite) TestDeleteManifestInMultiProjects() { } func TestMain(m *testing.M) { + config.Init() dao.PrepareTestForPostgresSQL() if result := m.Run(); result != 0 { diff --git a/src/core/middlewares/interceptor/quota/options.go b/src/core/middlewares/interceptor/quota/options.go index ca43c4165..ddf102a74 100644 --- a/src/core/middlewares/interceptor/quota/options.go +++ b/src/core/middlewares/interceptor/quota/options.go @@ -36,6 +36,8 @@ const ( // Options ... type Options struct { + enforceResources *bool + Action Action Manager *quota.Manager MutexKeys []string @@ -48,6 +50,15 @@ type Options struct { OnFinally func(http.ResponseWriter, *http.Request) error } +// EnforceResources ... +func (opts *Options) EnforceResources() bool { + return opts.enforceResources != nil && *opts.enforceResources +} + +func boolPtr(v bool) *bool { + return &v +} + func newOptions(opt ...Option) Options { opts := Options{} @@ -63,9 +74,20 @@ func newOptions(opt ...Option) Options { opts.StatusCode = http.StatusOK } + if opts.enforceResources == nil { + opts.enforceResources = boolPtr(true) + } + return opts } +// EnforceResources sets the interceptor enforceResources +func EnforceResources(enforceResources bool) Option { + return func(o *Options) { + o.enforceResources = boolPtr(enforceResources) + } +} + // WithAction sets the interceptor action func WithAction(a Action) Option { return func(o *Options) { diff --git a/src/core/middlewares/interceptor/quota/quota.go b/src/core/middlewares/interceptor/quota/quota.go index 85c289ff3..2914af8ee 100644 --- a/src/core/middlewares/interceptor/quota/quota.go +++ b/src/core/middlewares/interceptor/quota/quota.go @@ -49,28 +49,19 @@ func (qi *quotaInterceptor) HandleRequest(req *http.Request) (err error) { } }() - opts := qi.opts - - for _, key := range opts.MutexKeys { - m, err := redis.RequireLock(key) - if err != nil { - return err - } - qi.mutexes = append(qi.mutexes, m) + err = qi.requireMutexes() + if err != nil { + return } - resources := opts.Resources - if len(resources) == 0 && opts.OnResources != nil { - resources, err = opts.OnResources(req) - if err != nil { - return fmt.Errorf("failed to compute the resources for quota, error: %v", err) - } + err = qi.computeResources(req) + if err != nil { + return } - qi.resources = resources err = qi.reserve() if err != nil { - log.Errorf("Failed to %s resources, error: %v", opts.Action, err) + log.Errorf("Failed to %s resources, error: %v", qi.opts.Action, err) } return @@ -113,6 +104,23 @@ func (qi *quotaInterceptor) HandleResponse(w http.ResponseWriter, req *http.Requ } } +func (qi *quotaInterceptor) requireMutexes() error { + if !qi.opts.EnforceResources() { + // Do nothing for locks when quota interceptor not enforce resources + return nil + } + + for _, key := range qi.opts.MutexKeys { + m, err := redis.RequireLock(key) + if err != nil { + return err + } + qi.mutexes = append(qi.mutexes, m) + } + + return nil +} + func (qi *quotaInterceptor) freeMutexes() { for i := len(qi.mutexes) - 1; i >= 0; i-- { if err := redis.FreeLock(qi.mutexes[i]); err != nil { @@ -121,7 +129,30 @@ func (qi *quotaInterceptor) freeMutexes() { } } +func (qi *quotaInterceptor) computeResources(req *http.Request) error { + if !qi.opts.EnforceResources() { + // Do nothing in compute resources when quota interceptor not enforce resources + return nil + } + + if len(qi.opts.Resources) == 0 && qi.opts.OnResources != nil { + resources, err := qi.opts.OnResources(req) + if err != nil { + return fmt.Errorf("failed to compute the resources for quota, error: %v", err) + } + + qi.resources = resources + } + + return nil +} + func (qi *quotaInterceptor) reserve() error { + if !qi.opts.EnforceResources() { + // Do nothing in reserve resources when quota interceptor not enforce resources + return nil + } + if len(qi.resources) == 0 { return nil } @@ -137,6 +168,11 @@ func (qi *quotaInterceptor) reserve() error { } func (qi *quotaInterceptor) unreserve() error { + if !qi.opts.EnforceResources() { + // Do nothing in unreserve resources when quota interceptor not enforce resources + return nil + } + if len(qi.resources) == 0 { return nil } diff --git a/src/core/middlewares/sizequota/builder.go b/src/core/middlewares/sizequota/builder.go index 310c6e5bc..a6e1ecf92 100644 --- a/src/core/middlewares/sizequota/builder.go +++ b/src/core/middlewares/sizequota/builder.go @@ -22,6 +22,7 @@ import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares/interceptor" "github.com/goharbor/harbor/src/core/middlewares/interceptor/quota" "github.com/goharbor/harbor/src/core/middlewares/util" @@ -89,6 +90,7 @@ func (*blobStorageQuotaBuilder) Build(req *http.Request) (interceptor.Intercepto *req = *(req.WithContext(util.NewBlobInfoContext(req.Context(), info))) opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)), quota.WithAction(quota.AddAction), quota.StatusCode(http.StatusCreated), // NOTICE: mount blob and blob upload complete both return 201 when success @@ -119,6 +121,7 @@ func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Intercepto *req = *req.WithContext(util.NewManifestInfoContext(req.Context(), info)) opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)), quota.WithAction(quota.AddAction), quota.StatusCode(http.StatusCreated), @@ -181,6 +184,7 @@ func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Intercepto } opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)), quota.WithAction(quota.SubtractAction), quota.StatusCode(http.StatusAccepted), diff --git a/src/core/middlewares/sizequota/handler_test.go b/src/core/middlewares/sizequota/handler_test.go index cd9ca972f..e2b2bb309 100644 --- a/src/core/middlewares/sizequota/handler_test.go +++ b/src/core/middlewares/sizequota/handler_test.go @@ -30,8 +30,10 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest" "github.com/docker/distribution/manifest/schema2" + "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares/countquota" "github.com/goharbor/harbor/src/core/middlewares/util" "github.com/goharbor/harbor/src/pkg/types" @@ -662,7 +664,40 @@ func (suite *HandlerSuite) TestDeleteImageRace() { }) } +func (suite *HandlerSuite) TestDisableProjectQuota() { + withProject(func(projectID int64, projectName string) { + manifest := makeManifest(1, []int64{2, 3, 4, 5}) + pushImage(projectName, "photon", "latest", manifest) + + quotas, err := dao.ListQuotas(&models.QuotaQuery{ + Reference: "project", + ReferenceID: strconv.FormatInt(projectID, 10), + }) + + suite.Nil(err) + suite.Len(quotas, 1) + }) + + withProject(func(projectID int64, projectName string) { + cfg := config.GetCfgManager() + cfg.Set(common.QuotaPerProjectEnable, false) + defer cfg.Set(common.QuotaPerProjectEnable, true) + + manifest := makeManifest(1, []int64{2, 3, 4, 5}) + pushImage(projectName, "photon", "latest", manifest) + + quotas, err := dao.ListQuotas(&models.QuotaQuery{ + Reference: "project", + ReferenceID: strconv.FormatInt(projectID, 10), + }) + + suite.Nil(err) + suite.Len(quotas, 0) + }) +} + func TestMain(m *testing.M) { + config.Init() dao.PrepareTestForPostgresSQL() if result := m.Run(); result != 0 {