Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-22 18:25:56 +01:00
Code Issues Projects Releases Wiki Activity

refactor(quota): implement internal quota APIs by quota controller (#11058)

Browse Source 
1. Use quota controller to implement the internal quota APIs.
2. The internal quota APIs can exceed the quota limitations.

Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2020-03-16 10:20:17 +08:00 committed by GitHub
parent 42956c74bb
commit 60f8595034
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 311 additions and 1050 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/api/quota/controller.go
View File

@ -53,11 +53,14 @@ type Controller interface {
// Get returns quota by id
Get(ctx context.Context, id int64) (*quota.Quota, error)
// GetByRef returns quota by reference object
GetByRef(ctx context.Context, reference, referenceID string) (*quota.Quota, error)
// IsEnabled returns true when quota enabled for reference object
IsEnabled(ctx context.Context, reference, referenceID string) (bool, error)
// Refresh refresh quota for the reference object
Refresh(ctx context.Context, reference, referenceID string) error
Refresh(ctx context.Context, reference, referenceID string, options ...Option) error
// Request request resources to run f
// Before run the function, it reserves the resources,
@ -94,8 +97,12 @@ func (c *controller) Get(ctx context.Context, id int64) (*quota.Quota, error) {
return c.quotaMgr.Get(ctx, id)
}
func (c *controller) GetByRef(ctx context.Context, reference, referenceID string) (*quota.Quota, error) {
return c.quotaMgr.GetByRef(ctx, reference, referenceID)
}
func (c *controller) IsEnabled(ctx context.Context, reference, referenceID string) (bool, error) {
d, err := quotaDriver(ctx, reference, referenceID)
d, err := Driver(ctx, reference)
if err != nil {
return false, err
}
@ -139,7 +146,7 @@ func (c *controller) setReservedResources(ctx context.Context, reference, refere
func (c *controller) reserveResources(ctx context.Context, reference, referenceID string, resources types.ResourceList) error {
reserve := func(ctx context.Context) error {
q, err := c.quotaMgr.GetForUpdate(ctx, reference, referenceID)
q, err := c.quotaMgr.GetByRefForUpdate(ctx, reference, referenceID)
if err != nil {
return err
}
@ -163,7 +170,7 @@ func (c *controller) reserveResources(ctx context.Context, reference, referenceI
newReserved := types.Add(reserved, resources)
newUsed := types.Add(used, newReserved)
if err := quota.IsSafe(hardLimits, used, newUsed); err != nil {
if err := quota.IsSafe(hardLimits, used, newUsed, false); err != nil {
return ierror.DeniedError(nil).WithMessage("Quota exceeded when processing the request of %v", err)
}
@ -180,7 +187,7 @@ func (c *controller) reserveResources(ctx context.Context, reference, referenceI
func (c *controller) unreserveResources(ctx context.Context, reference, referenceID string, resources types.ResourceList) error {
unreserve := func(ctx context.Context) error {
if _, err := c.quotaMgr.GetForUpdate(ctx, reference, referenceID); err != nil {
if _, err := c.quotaMgr.GetByRefForUpdate(ctx, reference, referenceID); err != nil {
return err
}
@ -207,14 +214,16 @@ func (c *controller) unreserveResources(ctx context.Context, reference, referenc
return orm.WithTransaction(unreserve)(ctx)
}
func (c *controller) Refresh(ctx context.Context, reference, referenceID string) error {
driver, err := quotaDriver(ctx, reference, referenceID)
func (c *controller) Refresh(ctx context.Context, reference, referenceID string, options ...Option) error {
driver, err := Driver(ctx, reference)
if err != nil {
return err
}
opts := newOptions(options...)
refresh := func(ctx context.Context) error {
q, err := c.quotaMgr.GetForUpdate(ctx, reference, referenceID)
q, err := c.quotaMgr.GetByRefForUpdate(ctx, reference, referenceID)
if err != nil {
return err
}
@ -240,7 +249,7 @@ func (c *controller) Refresh(ctx context.Context, reference, referenceID string)
return fmt.Errorf("quota usage is negative for resource(s): %s", quota.PrettyPrintResourceNames(negativeUsed))
}
if err := quota.IsSafe(hardLimits, used, newUsed); err != nil {
if err := quota.IsSafe(hardLimits, used, newUsed, opts.IgnoreLimitation); err != nil {
return err
}
@ -277,7 +286,8 @@ func (c *controller) Request(ctx context.Context, reference, referenceID string,
return c.Refresh(ctx, reference, referenceID)
}
func quotaDriver(ctx context.Context, reference, referenceID string) (driver.Driver, error) {
// Driver returns quota driver for the reference
func Driver(ctx context.Context, reference string) (driver.Driver, error) {
d, ok := driver.Get(reference)
if !ok {
return nil, fmt.Errorf("quota not support for %s", reference)

6
src/api/quota/controller_test.go
View File

@ -70,7 +70,7 @@ func (suite *ControllerTestSuite) TestReserveResources() {
hardLimits := types.ResourceList{types.ResourceCount: 1}
mock.OnAnything(quotaMgr, "GetForUpdate").Return(&quota.Quota{Hard: hardLimits.String(), Used: types.Zero(hardLimits).String()}, nil)
mock.OnAnything(quotaMgr, "GetByRefForUpdate").Return(&quota.Quota{Hard: hardLimits.String(), Used: types.Zero(hardLimits).String()}, nil)
ctl := &controller{quotaMgr: quotaMgr, reservedExpiration: defaultReservedExpiration}
@ -88,7 +88,7 @@ func (suite *ControllerTestSuite) TestUnreserveResources() {
hardLimits := types.ResourceList{types.ResourceCount: 1}
mock.OnAnything(quotaMgr, "GetForUpdate").Return(&quota.Quota{Hard: hardLimits.String(), Used: types.Zero(hardLimits).String()}, nil)
mock.OnAnything(quotaMgr, "GetByRefForUpdate").Return(&quota.Quota{Hard: hardLimits.String(), Used: types.Zero(hardLimits).String()}, nil)
ctl := &controller{quotaMgr: quotaMgr, reservedExpiration: defaultReservedExpiration}
@ -113,7 +113,7 @@ func (suite *ControllerTestSuite) TestRequest() {
q := &quota.Quota{Hard: hardLimits.String(), Used: types.Zero(hardLimits).String()}
used := types.ResourceList{types.ResourceCount: 0}
mock.OnAnything(quotaMgr, "GetForUpdate").Return(q, nil)
mock.OnAnything(quotaMgr, "GetByRefForUpdate").Return(q, nil)
mock.OnAnything(quotaMgr, "Update").Return(nil).Run(func(mock.Arguments) {
q.SetUsed(used)

10
src/api/quota/driver/project/project.go
View File

@ -25,6 +25,7 @@ import (
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/config"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/q"
dr "github.com/goharbor/harbor/src/pkg/quota/driver"
"github.com/goharbor/harbor/src/pkg/types"
@ -45,10 +46,19 @@ type driver struct {
}
func (d *driver) Enabled(ctx context.Context, key string) (bool, error) {
// NOTE: every time load the new configurations from the db to get the latest configurations may have performance problem.
if err := d.cfg.Load(); err != nil {
return false, err
}
return d.cfg.Get(common.QuotaPerProjectEnable).GetBool(), nil
}
func (d *driver) HardLimits(ctx context.Context) types.ResourceList {
// NOTE: every time load the new configurations from the db to get the latest configurations may have performance problem.
if err := d.cfg.Load(); err != nil {
log.Warningf("load configurations failed, error: %v", err)
}
return types.ResourceList{
types.ResourceCount: d.cfg.Get(common.CountPerProject).GetInt64(),
types.ResourceStorage: d.cfg.Get(common.StoragePerProject).GetInt64(),

5
src/api/quota/driver/project/util.go
View File

@ -16,11 +16,11 @@ package project
import (
"context"
"fmt"
"strconv"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/graph-gophers/dataloader"
)
@ -70,7 +70,8 @@ func getProjectsBatchFn(ctx context.Context, keys dataloader.Keys) []*dataloader
for _, projectID := range projectIDs {
project, ok := projectsMap[projectID]
if !ok {
return handleError(fmt.Errorf("project not found, "+"project_id: %d", projectID))
err := ierror.NotFoundError(nil).WithMessage("project %d not found", projectID)
return handleError(err)
}
owner, ok := ownersMap[project.OwnerID]

38
src/api/quota/options.go Normal file
View File

@ -0,0 +1,38 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package quota
// Option option for `Refresh` method of `Controller`
type Option func(*Options)
// Options options used by `Refresh` method of `Controller`
type Options struct {
IgnoreLimitation bool
}
// IgnoreLimitation set IgnoreLimitation for the Options
func IgnoreLimitation(ignoreLimitation bool) func(*Options) {
return func(opts *Options) {
opts.IgnoreLimitation = ignoreLimitation
}
}
func newOptions(options ...Option) *Options {
opts := &Options{}
for _, f := range options {
f(opts)
}
return opts
}

11
src/common/models/project.go
View File

@ -192,6 +192,12 @@ func (p *Project) TableName() string {
return ProjectTable
}
// QuotaSummary ...
type QuotaSummary struct {
Hard types.ResourceList `json:"hard"`
Used types.ResourceList `json:"used"`
}
// ProjectSummary ...
type ProjectSummary struct {
RepoCount int64 `json:"repo_count"`
@ -203,8 +209,5 @@ type ProjectSummary struct {
GuestCount int64 `json:"guest_count"`
LimitedGuestCount int64 `json:"limited_guest_count"`
Quota struct {
Hard types.ResourceList `json:"hard"`
Used types.ResourceList `json:"used"`
} `json:"quota"`
Quota *QuotaSummary `json:"quota,omitempty"`
}

5
src/core/api/harborapi_test.go
View File

@ -37,7 +37,6 @@ import (
testutils "github.com/goharbor/harbor/src/common/utils/test"
api_models "github.com/goharbor/harbor/src/core/api/models"
apimodels "github.com/goharbor/harbor/src/core/api/models"
quota "github.com/goharbor/harbor/src/core/api/quota"
_ "github.com/goharbor/harbor/src/core/auth/db"
_ "github.com/goharbor/harbor/src/core/auth/ldap"
"github.com/goharbor/harbor/src/core/config"
@ -211,10 +210,6 @@ func init() {
beego.Router("/api/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner")
beego.Router("/api/projects/:pid([0-9]+)/scanner/candidates", proScannerAPI, "get:GetProScannerCandidates")
if err := quota.Sync(config.GlobalProjectMgr, false); err != nil {
log.Fatalf("failed to sync quota from backend: %v", err)
}
// Init user Info
admin = &usrInfo{adminName, adminPwd}
unknownUsr = &usrInfo{"unknown", "unknown"}

78
src/core/api/internal.go
View File

@ -15,18 +15,18 @@
package api
import (
"fmt"
"context"
o "github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/api/quota"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
common_quota "github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/jobservice/logger"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/internal/orm"
"github.com/pkg/errors"
"strconv"
quota "github.com/goharbor/harbor/src/core/api/quota"
)
// InternalAPI handles request of harbor admin...
@ -86,7 +86,9 @@ func (ia *InternalAPI) SwitchQuota() {
config.GetCfgManager().Set(common.ReadOnly, true)
config.GetCfgManager().Save()
}
if err := ia.ensureQuota(); err != nil {
ctx := orm.NewContext(ia.Ctx.Request.Context(), o.NewOrm())
if err := ia.refreshQuotas(ctx); err != nil {
ia.SendInternalServerError(err)
return
}
@ -99,50 +101,33 @@ func (ia *InternalAPI) SwitchQuota() {
return
}
func (ia *InternalAPI) ensureQuota() error {
func (ia *InternalAPI) refreshQuotas(ctx context.Context) error {
driver, err := quota.Driver(ctx, quota.ProjectReference)
if err != nil {
return err
}
projects, err := dao.GetProjects(nil)
if err != nil {
return err
}
for _, project := range projects {
pSize, err := dao.CountSizeOfProject(project.ProjectID)
if err != nil {
logger.Warningf("error happen on counting size of project:%d , error:%v, just skip it.", project.ProjectID, err)
continue
}
afQuery := &models.ArtifactQuery{
PID: project.ProjectID,
}
afs, err := dao.ListArtifacts(afQuery)
if err != nil {
logger.Warningf("error happen on counting number of project:%d , error:%v, just skip it.", project.ProjectID, err)
continue
}
pCount := int64(len(afs))
// it needs to append the chart count
if config.WithChartMuseum() {
count, err := chartController.GetCountOfCharts([]string{project.Name})
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("get chart count of project %d failed", project.ProjectID))
logger.Error(err)
for _, project := range projects {
referenceID := quota.ReferenceID(project.ProjectID)
_, err := quota.Ctl.GetByRef(ctx, quota.ProjectReference, referenceID)
if ierror.IsNotFoundErr(err) {
if _, err := quota.Ctl.Create(ctx, quota.ProjectReference, referenceID, driver.HardLimits(ctx)); err != nil {
log.Warningf("initialize quota for project %s failed, error: %v", project.Name, err)
continue
}
pCount = pCount + int64(count)
} else if err != nil {
log.Warningf("get quota of the project %s failed, error: %v", project.Name, err)
continue
}
quotaMgr, err := common_quota.NewManager("project", strconv.FormatInt(project.ProjectID, 10))
if err != nil {
logger.Errorf("Error occurred when to new quota manager %v, just skip it.", err)
continue
}
used := common_quota.ResourceList{
common_quota.ResourceStorage: pSize,
common_quota.ResourceCount: pCount,
}
if err := quotaMgr.EnsureQuota(used); err != nil {
logger.Errorf("cannot ensure quota for the project: %d, err: %v, just skip it.", project.ProjectID, err)
continue
if err := quota.Ctl.Refresh(ctx, quota.ProjectReference, referenceID, quota.IgnoreLimitation(true)); err != nil {
log.Warningf("refresh quota usage for project %s failed, error: %v", project.Name, err)
}
}
return nil
@ -150,6 +135,11 @@ func (ia *InternalAPI) ensureQuota() error {
// SyncQuota ...
func (ia *InternalAPI) SyncQuota() {
if !config.QuotaPerProjectEnable() {
ia.SendError(ierror.ForbiddenError(nil).WithMessage("quota per project is disabled"))
return
}
cur := config.ReadOnly()
cfgMgr := config.GetCfgManager()
if !cur {
@ -163,8 +153,8 @@ func (ia *InternalAPI) SyncQuota() {
cfgMgr.Save()
}()
log.Info("start to sync quota(API), the system will be set to ReadOnly and back it normal once it done.")
// As the sync function ignores all of duplicate error, it's safe to enable persist DB.
err := quota.Sync(ia.ProjectMgr, true)
ctx := orm.NewContext(context.TODO(), o.NewOrm())
err := ia.refreshQuotas(ctx)
if err != nil {
log.Errorf("fail to sync quota(API), but with error: %v, please try to do it again.", err)
return

12
src/core/api/project.go
View File

@ -16,6 +16,12 @@ package api
import (
"fmt"
"net/http"
"regexp"
"strconv"
"strings"
"sync"
"github.com/goharbor/harbor/src/api/event"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
@ -31,11 +37,6 @@ import (
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/goharbor/harbor/src/pkg/types"
"github.com/pkg/errors"
"net/http"
"regexp"
"strconv"
"strings"
"sync"
)
type deletableResp struct {
@ -657,6 +658,7 @@ func getProjectQuotaSummary(projectID int64, summary *models.ProjectSummary) {
quota := quotas[0]
summary.Quota = &models.QuotaSummary{}
summary.Quota.Hard, _ = types.NewResourceList(quota.Hard)
summary.Quota.Used, _ = types.NewResourceList(quota.Used)
}

226
src/core/api/quota/chart/chart.go
View File

@ -1,226 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chart
import (
"fmt"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
common_quota "github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api"
quota "github.com/goharbor/harbor/src/core/api/quota"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
"github.com/pkg/errors"
"net/url"
"strings"
"sync"
)
// Migrator ...
type Migrator struct {
pm promgr.ProjectManager
}
// NewChartMigrator returns a new RegistryMigrator.
func NewChartMigrator(pm promgr.ProjectManager) quota.QuotaMigrator {
migrator := Migrator{
pm: pm,
}
return &migrator
}
var (
controller *chartserver.Controller
controllerErr error
controllerOnce sync.Once
)
// Ping ...
func (rm *Migrator) Ping() error {
return quota.Check(api.HealthCheckerRegistry["chartmuseum"].Check)
}
// Dump ...
// Depends on DB to dump chart data, as chart cannot get all of namespaces.
func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
var (
projects []quota.ProjectInfo
wg sync.WaitGroup
err error
)
all, err := dao.GetProjects(nil)
if err != nil {
return nil, err
}
wg.Add(len(all))
errChan := make(chan error, 1)
infoChan := make(chan interface{})
done := make(chan bool, 1)
go func() {
defer func() {
done <- true
}()
for {
select {
case result := <-infoChan:
if result == nil {
return
}
project, ok := result.(quota.ProjectInfo)
if ok {
projects = append(projects, project)
}
case e := <-errChan:
if err == nil {
err = errors.Wrap(e, "quota sync error on getting info of project")
} else {
err = errors.Wrap(e, err.Error())
}
}
}
}()
for _, project := range all {
go func(project *models.Project) {
defer wg.Done()
var repos []quota.RepoData
ctr, err := chartController()
if err != nil {
errChan <- err
return
}
chartInfo, err := ctr.ListCharts(project.Name)
if err != nil {
errChan <- err
return
}
// repo
for _, chart := range chartInfo {
var afs []*models.Artifact
chartVersions, err := ctr.GetChart(project.Name, chart.Name)
if err != nil {
errChan <- err
continue
}
for _, chart := range chartVersions {
af := &models.Artifact{
PID: project.ProjectID,
Repo: chart.Name,
Tag: chart.Version,
Digest: chart.Digest,
Kind: "Chart",
}
afs = append(afs, af)
}
repoData := quota.RepoData{
Name: project.Name,
Afs: afs,
}
repos = append(repos, repoData)
}
projectInfo := quota.ProjectInfo{
Name: project.Name,
Repos: repos,
}
infoChan <- projectInfo
}(project)
}
wg.Wait()
close(infoChan)
<-done
if err != nil {
return nil, err
}
return projects, nil
}
// Usage ...
// Chart will not cover size.
func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) {
var pros []quota.ProjectUsage
for _, project := range projects {
var count int64
// usage count
for _, repo := range project.Repos {
count = count + int64(len(repo.Afs))
}
proUsage := quota.ProjectUsage{
Project: project.Name,
Used: common_quota.ResourceList{
common_quota.ResourceCount: count,
common_quota.ResourceStorage: 0,
},
}
pros = append(pros, proUsage)
}
return pros, nil
}
// Persist ...
// Chart will not persist data into db.
func (rm *Migrator) Persist(projects []quota.ProjectInfo) error {
return nil
}
func chartController() (*chartserver.Controller, error) {
controllerOnce.Do(func() {
addr, err := config.GetChartMuseumEndpoint()
if err != nil {
controllerErr = fmt.Errorf("failed to get the endpoint URL of chart storage server: %s", err.Error())
return
}
addr = strings.TrimSuffix(addr, "/")
url, err := url.Parse(addr)
if err != nil {
controllerErr = errors.New("endpoint URL of chart storage server is malformed")
return
}
ctr, err := chartserver.NewController(url)
if err != nil {
controllerErr = errors.New("failed to initialize chart API controller")
}
controller = ctr
log.Debugf("Chart storage server is set to %s", url.String())
log.Info("API controller for chart repository server is successfully initialized")
})
return controller, controllerErr
}
func init() {
quota.Register("chart", NewChartMigrator)
}

199
src/core/api/quota/migrator.go
View File

@ -1,199 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
"github.com/goharbor/harbor/src/pkg/types"
"strconv"
"time"
)
// QuotaMigrator ...
type QuotaMigrator interface {
// Ping validates and wait for backend service ready.
Ping() error
// Dump exports all data from backend service, registry, chartmuseum
Dump() ([]ProjectInfo, error)
// Usage computes the quota usage of all the projects
Usage([]ProjectInfo) ([]ProjectUsage, error)
// Persist record the data to DB, artifact, artifact_blob and blob table.
Persist([]ProjectInfo) error
}
// ProjectInfo ...
type ProjectInfo struct {
Name string
Repos []RepoData
}
// RepoData ...
type RepoData struct {
Name string
Afs []*models.Artifact
Afnbs []*models.ArtifactAndBlob
Blobs []*models.Blob
}
// ProjectUsage ...
type ProjectUsage struct {
Project string
Used quota.ResourceList
}
// Instance ...
type Instance func(promgr.ProjectManager) QuotaMigrator
var adapters = make(map[string]Instance)
// Register ...
func Register(name string, adapter Instance) {
if adapter == nil {
panic("quota: Register adapter is nil")
}
if _, ok := adapters[name]; ok {
panic("quota: Register called twice for adapter " + name)
}
adapters[name] = adapter
}
// Sync ...
func Sync(pm promgr.ProjectManager, populate bool) error {
totalUsage := make(map[string][]ProjectUsage)
for name, instanceFunc := range adapters {
if !config.WithChartMuseum() {
if name == "chart" {
continue
}
}
adapter := instanceFunc(pm)
log.Infof("[Quota-Sync]:: start to ping server ... [%s]", name)
if err := adapter.Ping(); err != nil {
log.Infof("[Quota-Sync]:: fail to ping server ... [%s], quit sync ...", name)
return err
}
log.Infof("[Quota-Sync]:: success to ping server ... [%s]", name)
log.Infof("[Quota-Sync]:: start to dump data from server ... [%s]", name)
data, err := adapter.Dump()
if err != nil {
log.Infof("[Quota-Sync]:: fail to dump data from server ... [%s], quit sync ...", name)
return err
}
log.Infof("[Quota-Sync]:: success to dump data from server ... [%s]", name)
usage, err := adapter.Usage(data)
if err != nil {
return err
}
totalUsage[name] = usage
if populate {
log.Infof("[Quota-Sync]:: start to persist data for server ... [%s]", name)
if err := adapter.Persist(data); err != nil {
log.Infof("[Quota-Sync]:: fail to persist data from server ... [%s], quit sync ...", name)
return err
}
log.Infof("[Quota-Sync]:: success to persist data for server ... [%s]", name)
}
}
merged := mergeUsage(totalUsage)
if err := ensureQuota(merged); err != nil {
return err
}
return nil
}
// Check ...
func Check(f func() error) error {
return retry(10, 2*time.Second, f)
}
// mergeUsage merges the usage of adapters
func mergeUsage(total map[string][]ProjectUsage) []ProjectUsage {
if !config.WithChartMuseum() {
return total["registry"]
}
regUsgs := total["registry"]
chartUsgs := total["chart"]
var mergedUsage []ProjectUsage
temp := make(map[string]quota.ResourceList)
for _, regUsg := range regUsgs {
_, exist := temp[regUsg.Project]
if !exist {
temp[regUsg.Project] = regUsg.Used
mergedUsage = append(mergedUsage, ProjectUsage{
Project: regUsg.Project,
Used: regUsg.Used,
})
}
}
for _, chartUsg := range chartUsgs {
var usedTemp quota.ResourceList
_, exist := temp[chartUsg.Project]
if !exist {
usedTemp = chartUsg.Used
} else {
usedTemp = types.Add(temp[chartUsg.Project], chartUsg.Used)
}
temp[chartUsg.Project] = usedTemp
mergedUsage = append(mergedUsage, ProjectUsage{
Project: chartUsg.Project,
Used: usedTemp,
})
}
return mergedUsage
}
// ensureQuota updates the quota and quota usage in the data base.
func ensureQuota(usages []ProjectUsage) error {
var pid int64
for _, usage := range usages {
project, err := dao.GetProjectByName(usage.Project)
if err != nil {
log.Error(err)
return err
}
pid = project.ProjectID
quotaMgr, err := quota.NewManager("project", strconv.FormatInt(pid, 10))
if err != nil {
log.Errorf("Error occurred when to new quota manager %v", err)
return err
}
if err := quotaMgr.EnsureQuota(usage.Used); err != nil {
log.Errorf("cannot ensure quota for the project: %d, err: %v", pid, err)
return err
}
}
return nil
}
func retry(attempts int, sleep time.Duration, f func() error) error {
if err := f(); err != nil {
if attempts--; attempts > 0 {
time.Sleep(sleep)
return retry(attempts, sleep, f)
}
return err
}
return nil
}

460
src/core/api/quota/registry/registry.go
View File

@ -1,460 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
common_quota "github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api"
quota "github.com/goharbor/harbor/src/core/api/quota"
"github.com/goharbor/harbor/src/core/promgr"
"github.com/goharbor/harbor/src/pkg/registry"
"github.com/pkg/errors"
"strings"
"sync"
"time"
)
// Migrator ...
type Migrator struct {
pm promgr.ProjectManager
}
// NewRegistryMigrator returns a new Migrator.
func NewRegistryMigrator(pm promgr.ProjectManager) quota.QuotaMigrator {
migrator := Migrator{
pm: pm,
}
return &migrator
}
// Ping ...
func (rm *Migrator) Ping() error {
return quota.Check(api.HealthCheckerRegistry["registry"].Check)
}
// Dump ...
func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
var (
projects []quota.ProjectInfo
wg sync.WaitGroup
err error
)
reposInRegistry, err := registry.Cli.Catalog()
if err != nil {
return nil, err
}
// repoMap : map[project_name : []repo list]
repoMap := make(map[string][]string)
for _, item := range reposInRegistry {
projectName := strings.Split(item, "/")[0]
pro, err := rm.pm.Get(projectName)
if err != nil {
log.Errorf("failed to get project %s: %v", projectName, err)
continue
}
if pro == nil {
continue
}
_, exist := repoMap[pro.Name]
if !exist {
repoMap[pro.Name] = []string{item}
} else {
repos := repoMap[pro.Name]
repos = append(repos, item)
repoMap[pro.Name] = repos
}
}
repoMap, err = rm.appendEmptyProject(repoMap)
if err != nil {
log.Errorf("fail to add empty projects: %v", err)
return nil, err
}
wg.Add(len(repoMap))
errChan := make(chan error, 1)
infoChan := make(chan interface{})
done := make(chan bool, 1)
go func() {
defer func() {
done <- true
}()
for {
select {
case result := <-infoChan:
if result == nil {
return
}
project, ok := result.(quota.ProjectInfo)
if ok {
projects = append(projects, project)
}
case e := <-errChan:
if err == nil {
err = errors.Wrap(e, "quota sync error on getting info of project")
} else {
err = errors.Wrap(e, err.Error())
}
}
}
}()
for project, repos := range repoMap {
go func(project string, repos []string) {
defer wg.Done()
info, err := infoOfProject(project, repos)
if err != nil {
errChan <- err
return
}
infoChan <- info
}(project, repos)
}
wg.Wait()
close(infoChan)
// wait for all of project info
<-done
if err != nil {
return nil, err
}
return projects, nil
}
// As catalog api cannot list the empty projects in harbor, here it needs to append the empty projects into repo infor
// so that quota syncer can add 0 usage into quota usage.
func (rm *Migrator) appendEmptyProject(repoMap map[string][]string) (map[string][]string, error) {
var withEmptyProjects map[string][]string
all, err := dao.GetProjects(nil)
if err != nil {
return withEmptyProjects, err
}
withEmptyProjects = repoMap
for _, pro := range all {
_, exist := repoMap[pro.Name]
if !exist {
withEmptyProjects[pro.Name] = []string{}
}
}
return withEmptyProjects, nil
}
// Usage ...
// registry needs to merge the shard blobs of different repositories.
func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) {
var pros []quota.ProjectUsage
for _, project := range projects {
var size, count int64
var blobs = make(map[string]int64)
// usage count
for _, repo := range project.Repos {
count = count + int64(len(repo.Afs))
// Because that there are some shared blobs between repositories, it needs to remove the duplicate items.
for _, blob := range repo.Blobs {
_, exist := blobs[blob.Digest]
// foreign blob won't be calculated
if !exist && blob.ContentType != common.ForeignLayer {
blobs[blob.Digest] = blob.Size
}
}
}
// size
for _, item := range blobs {
size = size + item
}
proUsage := quota.ProjectUsage{
Project: project.Name,
Used: common_quota.ResourceList{
common_quota.ResourceCount: count,
common_quota.ResourceStorage: size,
},
}
pros = append(pros, proUsage)
}
return pros, nil
}
// Persist ...
func (rm *Migrator) Persist(projects []quota.ProjectInfo) error {
total := len(projects)
for i, project := range projects {
log.Infof("[Quota-Sync]:: start to persist artifact&blob for project: %s, progress... [%d/%d]", project.Name, i, total)
for _, repo := range project.Repos {
if err := persistAf(repo.Afs); err != nil {
return err
}
if err := persistAfnbs(repo.Afnbs); err != nil {
return err
}
if err := persistBlob(repo.Blobs); err != nil {
return err
}
}
log.Infof("[Quota-Sync]:: success to persist artifact&blob for project: %s, progress... [%d/%d]", project.Name, i, total)
}
if err := persistPB(projects); err != nil {
return err
}
return nil
}
func persistAf(afs []*models.Artifact) error {
if len(afs) != 0 {
for _, af := range afs {
_, err := dao.AddArtifact(af)
if err != nil {
if err == dao.ErrDupRows {
continue
}
log.Error(err)
return err
}
}
}
return nil
}
func persistAfnbs(afnbs []*models.ArtifactAndBlob) error {
if len(afnbs) != 0 {
for _, afnb := range afnbs {
_, err := dao.AddArtifactNBlob(afnb)
if err != nil {
if err == dao.ErrDupRows {
continue
}
log.Error(err)
return err
}
}
}
return nil
}
func persistBlob(blobs []*models.Blob) error {
if len(blobs) != 0 {
for _, blob := range blobs {
_, err := dao.AddBlob(blob)
if err != nil {
if err == dao.ErrDupRows {
continue
}
log.Error(err)
return err
}
}
}
return nil
}
func persistPB(projects []quota.ProjectInfo) error {
total := len(projects)
for i, project := range projects {
log.Infof("[Quota-Sync]:: start to persist project&blob for project: %s, progress... [%d/%d]", project.Name, i, total)
var blobs = make(map[string]int64)
var blobsOfPro []*models.Blob
for _, repo := range project.Repos {
for _, blob := range repo.Blobs {
_, exist := blobs[blob.Digest]
if exist {
continue
}
blobs[blob.Digest] = blob.Size
blobInDB, err := dao.GetBlob(blob.Digest)
if err != nil {
log.Error(err)
return err
}
if blobInDB != nil {
blobsOfPro = append(blobsOfPro, blobInDB)
}
}
}
pro, err := dao.GetProjectByName(project.Name)
if err != nil {
log.Error(err)
return err
}
_, err = dao.AddBlobsToProject(pro.ProjectID, blobsOfPro...)
if err != nil {
if err == dao.ErrDupRows {
continue
}
log.Error(err)
return err
}
log.Infof("[Quota-Sync]:: success to persist project&blob for project: %s, progress... [%d/%d]", project.Name, i, total)
}
return nil
}
func infoOfProject(project string, repoList []string) (quota.ProjectInfo, error) {
var (
repos []quota.RepoData
wg sync.WaitGroup
err error
)
wg.Add(len(repoList))
errChan := make(chan error, 1)
infoChan := make(chan interface{})
done := make(chan bool, 1)
pro, err := dao.GetProjectByName(project)
if err != nil {
log.Error(err)
return quota.ProjectInfo{}, err
}
go func() {
defer func() {
done <- true
}()
for {
select {
case result := <-infoChan:
if result == nil {
return
}
repoData, ok := result.(quota.RepoData)
if ok {
repos = append(repos, repoData)
}
case e := <-errChan:
if err == nil {
err = errors.Wrap(e, "quota sync error on getting info of repo")
} else {
err = errors.Wrap(e, err.Error())
}
}
}
}()
for _, repo := range repoList {
go func(pid int64, repo string) {
defer func() {
wg.Done()
}()
info, err := infoOfRepo(pid, repo)
if err != nil {
errChan <- err
return
}
infoChan <- info
}(pro.ProjectID, repo)
}
wg.Wait()
close(infoChan)
<-done
if err != nil {
return quota.ProjectInfo{}, err
}
return quota.ProjectInfo{
Name: project,
Repos: repos,
}, nil
}
func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
tags, err := registry.Cli.ListTags(repo)
if err != nil {
return quota.RepoData{}, err
}
var afnbs []*models.ArtifactAndBlob
var afs []*models.Artifact
var blobs []*models.Blob
for _, tag := range tags {
manifest, digest, err := registry.Cli.PullManifest(repo, tag)
if err != nil {
log.Error(err)
// To workaround issue: https://github.com/goharbor/harbor/issues/9299, just log the error and do not raise it.
// Let the sync process pass, but the 'Unknown manifest' will not be counted into size and count of quota usage.
// User still can view there images with size 0 in harbor.
continue
}
mediaType, payload, err := manifest.Payload()
if err != nil {
return quota.RepoData{}, err
}
// self
afnb := &models.ArtifactAndBlob{
DigestAF: digest,
DigestBlob: digest,
}
afnbs = append(afnbs, afnb)
// add manifest as a blob.
blob := &models.Blob{
Digest: digest,
ContentType: mediaType,
Size: int64(len(payload)),
CreationTime: time.Now(),
}
blobs = append(blobs, blob)
for _, layer := range manifest.References() {
afnb := &models.ArtifactAndBlob{
DigestAF: digest,
DigestBlob: layer.Digest.String(),
}
afnbs = append(afnbs, afnb)
blob := &models.Blob{
Digest: layer.Digest.String(),
ContentType: layer.MediaType,
Size: layer.Size,
CreationTime: time.Now(),
}
blobs = append(blobs, blob)
}
af := &models.Artifact{
PID: pid,
Repo: repo,
Tag: tag,
Digest: digest,
Kind: "Docker-Image",
CreationTime: time.Now(),
}
afs = append(afs, af)
}
return quota.RepoData{
Name: repo,
Afs: afs,
Afnbs: afnbs,
Blobs: blobs,
}, nil
}
func init() {
quota.Register("registry", NewRegistryMigrator)
}

71
src/core/main.go
View File

@ -17,10 +17,8 @@ package main
import (
"encoding/gob"
"fmt"
"github.com/goharbor/harbor/src/migration"
"os"
"os/signal"
"strconv"
"syscall"
"time"
@ -30,13 +28,9 @@ import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
common_quota "github.com/goharbor/harbor/src/common/quota"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api"
quota "github.com/goharbor/harbor/src/core/api/quota"
_ "github.com/goharbor/harbor/src/core/api/quota/chart"
_ "github.com/goharbor/harbor/src/core/api/quota/registry"
_ "github.com/goharbor/harbor/src/core/auth/authproxy"
_ "github.com/goharbor/harbor/src/core/auth/db"
_ "github.com/goharbor/harbor/src/core/auth/ldap"
@ -46,13 +40,13 @@ import (
"github.com/goharbor/harbor/src/core/filter"
"github.com/goharbor/harbor/src/core/middlewares"
"github.com/goharbor/harbor/src/core/service/token"
"github.com/goharbor/harbor/src/migration"
"github.com/goharbor/harbor/src/pkg/notification"
_ "github.com/goharbor/harbor/src/pkg/notifier/topic"
"github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/event"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/types"
"github.com/goharbor/harbor/src/pkg/version"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/server"
@ -88,69 +82,6 @@ func updateInitPassword(userID int, password string) error {
return nil
}
// Quota migration
func quotaSync() error {
projects, err := dao.GetProjects(nil)
if err != nil {
log.Errorf("list project error, %v", err)
return err
}
var pids []string
for _, project := range projects {
pids = append(pids, strconv.FormatInt(project.ProjectID, 10))
}
usages, err := dao.ListQuotaUsages(&models.QuotaUsageQuery{Reference: "project", ReferenceIDs: pids})
if err != nil {
log.Errorf("list quota usage error, %v", err)
return err
}
// The condition handles these two cases:
// 1, len(project) > 1 && len(usages) == 1. existing projects without usage, as we do always has 'library' usage in DB.
// 2, migration fails at the phase of inserting usage into DB, and parts of them are inserted successfully.
if len(projects) != len(usages) {
log.Info("Start to sync quota data .....")
if err := quota.Sync(config.GlobalProjectMgr, true); err != nil {
log.Errorf("Fail to sync quota data, %v", err)
return err
}
log.Info("Success to sync quota data .....")
return nil
}
// Only has one project without usage
zero := common_quota.ResourceList{
common_quota.ResourceCount: 0,
common_quota.ResourceStorage: 0,
}
if len(projects) == 1 && len(usages) == 1 {
totalRepo, err := dao.GetTotalOfRepositories()
if totalRepo == 0 {
return nil
}
refID, err := strconv.ParseInt(usages[0].ReferenceID, 10, 64)
if err != nil {
log.Error(err)
return err
}
usedRes, err := types.NewResourceList(usages[0].Used)
if err != nil {
log.Error(err)
return err
}
if types.Equals(usedRes, zero) && refID == projects[0].ProjectID {
log.Info("Start to sync quota data .....")
if err := quota.Sync(config.GlobalProjectMgr, true); err != nil {
log.Errorf("Fail to sync quota data, %v", err)
return err
}
log.Info("Success to sync quota data .....")
}
}
return nil
}
func gracefulShutdown(closing, done chan struct{}) {
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)

28
src/pkg/quota/dao/dao.go
View File

@ -34,8 +34,11 @@ type DAO interface {
// Get returns quota by id
Get(ctx context.Context, id int64) (*models.Quota, error)
// GetForUpdate get quota by reference object and lock it for update
GetForUpdate(ctx context.Context, reference, referenceID string) (*models.Quota, error)
// GetByRef returns quota by reference object
GetByRef(ctx context.Context, reference, referenceID string) (*models.Quota, error)
// GetByRefForUpdate get quota by reference object and lock it for update
GetByRefForUpdate(ctx context.Context, reference, referenceID string) (*models.Quota, error)
// Update update quota
Update(ctx context.Context, quota *models.Quota) error
@ -124,7 +127,26 @@ func (d *dao) Get(ctx context.Context, id int64) (*models.Quota, error) {
return toQuota(quota, usage), nil
}
func (d *dao) GetForUpdate(ctx context.Context, reference, referenceID string) (*models.Quota, error) {
func (d *dao) GetByRef(ctx context.Context, reference, referenceID string) (*models.Quota, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
quota := &Quota{Reference: reference, ReferenceID: referenceID}
if err := o.Read(quota, "reference", "reference_id"); err != nil {
return nil, orm.WrapNotFoundError(err, "quota not found for (%s, %s)", reference, referenceID)
}
usage := &QuotaUsage{Reference: reference, ReferenceID: referenceID}
if err := o.Read(usage, "reference", "reference_id"); err != nil {
return nil, orm.WrapNotFoundError(err, "quota usage not found for (%s, %s)", reference, referenceID)
}
return toQuota(quota, usage), nil
}
func (d *dao) GetByRefForUpdate(ctx context.Context, reference, referenceID string) (*models.Quota, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return nil, err

69
src/pkg/quota/dao/dao_test.go
View File

@ -15,8 +15,11 @@
package dao
import (
"context"
"sync"
"testing"
"github.com/goharbor/harbor/src/internal/orm"
"github.com/goharbor/harbor/src/pkg/types"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/suite"
@ -73,11 +76,75 @@ func (suite *DaoTestSuite) TestDelete() {
}
}
func (suite *DaoTestSuite) TestGetByRef() {
hardLimits := types.ResourceList{types.ResourceCount: 1}
usage := types.ResourceList{types.ResourceCount: 0}
reference, referenceID := "project", "4"
id, err := suite.dao.Create(suite.Context(), reference, referenceID, hardLimits, usage)
suite.Nil(err)
{
q, err := suite.dao.GetByRef(suite.Context(), reference, referenceID)
suite.Nil(err)
suite.NotNil(q)
}
suite.Nil(suite.dao.Delete(suite.Context(), id))
{
_, err := suite.dao.GetByRef(suite.Context(), reference, referenceID)
suite.Error(err)
}
}
func (suite *DaoTestSuite) TestGetByRefForUpdate() {
hardLimits := types.ResourceList{types.ResourceCount: 1}
usage := types.ResourceList{types.ResourceCount: 0}
reference, referenceID := "project", "5"
id, err := suite.dao.Create(suite.Context(), reference, referenceID, hardLimits, usage)
suite.Nil(err)
var wg sync.WaitGroup
count := int64(10)
for i := int64(0); i < count; i++ {
wg.Add(1)
go func() {
defer wg.Done()
f := func(ctx context.Context) error {
q, err := suite.dao.GetByRefForUpdate(ctx, reference, referenceID)
suite.Nil(err)
used, _ := q.GetUsed()
used[types.ResourceCount]++
q.SetUsed(used)
suite.dao.Update(ctx, q)
return nil
}
orm.WithTransaction(f)(suite.Context())
}()
}
wg.Wait()
{
q, err := suite.dao.Get(suite.Context(), id)
suite.Nil(err)
used, _ := q.GetUsed()
suite.Equal(count, used[types.ResourceCount])
}
}
func (suite *DaoTestSuite) TestUpdate() {
hardLimits := types.ResourceList{types.ResourceCount: 1}
usage := types.ResourceList{types.ResourceCount: 0}
id, err := suite.dao.Create(suite.Context(), "project", "3", hardLimits, usage)
id, err := suite.dao.Create(suite.Context(), "project", "6", hardLimits, usage)
suite.Nil(err)
newHardLimits := types.ResourceList{types.ResourceCount: 2}

15
src/pkg/quota/manager.go
View File

@ -37,8 +37,11 @@ type Manager interface {
// Get returns quota by id
Get(ctx context.Context, id int64) (*Quota, error)
// GetForUpdate returns quota by reference and reference id for update
GetForUpdate(ctx context.Context, reference, referenceID string) (*Quota, error)
// GetByRef returns quota by reference object
GetByRef(ctx context.Context, reference, referenceID string) (*Quota, error)
// GetByRefForUpdate returns quota by reference and reference id for update
GetByRefForUpdate(ctx context.Context, reference, referenceID string) (*Quota, error)
// Update update quota
Update(ctx context.Context, quota *Quota) error
@ -80,8 +83,12 @@ func (m *manager) Get(ctx context.Context, id int64) (*Quota, error) {
return m.dao.Get(ctx, id)
}
func (m *manager) GetForUpdate(ctx context.Context, reference, referenceID string) (*Quota, error) {
return m.dao.GetForUpdate(ctx, reference, referenceID)
func (m *manager) GetByRef(ctx context.Context, reference, referenceID string) (*Quota, error) {
return m.dao.GetByRef(ctx, reference, referenceID)
}
func (m *manager) GetByRefForUpdate(ctx context.Context, reference, referenceID string) (*Quota, error) {
return m.dao.GetByRefForUpdate(ctx, reference, referenceID)
}
func (m *manager) Update(ctx context.Context, q *Quota) error {

4
src/pkg/quota/util.go
View File

@ -22,7 +22,7 @@ import (
)
// IsSafe check new used is safe under the hard limits
func IsSafe(hardLimits types.ResourceList, currentUsed types.ResourceList, newUsed types.ResourceList) error {
func IsSafe(hardLimits types.ResourceList, currentUsed types.ResourceList, newUsed types.ResourceList, ignoreLimitation bool) error {
var errs Errors
for resource, value := range newUsed {
@ -36,7 +36,7 @@ func IsSafe(hardLimits types.ResourceList, currentUsed types.ResourceList, newUs
continue
}
if value > hardLimit {
if value > hardLimit && !ignoreLimitation {
errs = errs.Add(NewResourceOverflowError(resource, hardLimit, currentUsed[resource], value))
}
}

23
src/pkg/quota/util_test.go
View File

@ -22,9 +22,10 @@ import (
func TestIsSafe(t *testing.T) {
type args struct {
hardLimits types.ResourceList
currentUsed types.ResourceList
newUsed types.ResourceList
hardLimits types.ResourceList
currentUsed types.ResourceList
newUsed types.ResourceList
ignoreLimitation bool
}
tests := []struct {
name string
@ -37,6 +38,7 @@ func TestIsSafe(t *testing.T) {
types.ResourceList{types.ResourceStorage: types.UNLIMITED},
types.ResourceList{types.ResourceStorage: 1000},
types.ResourceList{types.ResourceStorage: 1000},
false,
},
false,
},
@ -46,6 +48,7 @@ func TestIsSafe(t *testing.T) {
types.ResourceList{types.ResourceStorage: 100},
types.ResourceList{types.ResourceStorage: 10},
types.ResourceList{types.ResourceStorage: 1},
false,
},
false,
},
@ -55,22 +58,34 @@ func TestIsSafe(t *testing.T) {
types.ResourceList{types.ResourceStorage: 100},
types.ResourceList{types.ResourceStorage: 0},
types.ResourceList{types.ResourceStorage: 200},
false,
},
true,
},
{
"ignore limitation",
args{
types.ResourceList{types.ResourceStorage: 100},
types.ResourceList{types.ResourceStorage: 0},
types.ResourceList{types.ResourceStorage: 200},
true,
},
false,
},
{
"hard limit not found",
args{
types.ResourceList{types.ResourceStorage: 100},
types.ResourceList{types.ResourceCount: 0},
types.ResourceList{types.ResourceCount: 1},
false,
},
true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := IsSafe(tt.args.hardLimits, tt.args.currentUsed, tt.args.newUsed); (err != nil) != tt.wantErr {
if err := IsSafe(tt.args.hardLimits, tt.args.currentUsed, tt.args.newUsed, tt.args.ignoreLimitation); (err != nil) != tt.wantErr {
t.Errorf("IsSafe() error = %v, wantErr %v", err, tt.wantErr)
}
})

2
src/portal/src/app/project/summary/summary.component.html
View File

@ -23,7 +23,7 @@
</ul>
</div>
</div>
<div *ngIf="showQuotaInfo" class="summary-right pt-1">
<div *ngIf="showQuotaInfo && summaryInformation?.quota" class="summary-right pt-1">
<div class="display-flex project-detail">
<h5 class="mt-0">{{'SUMMARY.PROJECT_QUOTAS' | translate}}</h5>
<div class="ml-1">

42
src/testing/api/quota/controller.go
View File

@ -8,6 +8,8 @@ import (
models "github.com/goharbor/harbor/src/pkg/quota/models"
mock "github.com/stretchr/testify/mock"
quota "github.com/goharbor/harbor/src/api/quota"
types "github.com/goharbor/harbor/src/pkg/types"
)
@ -81,6 +83,29 @@ func (_m *Controller) Get(ctx context.Context, id int64) (*models.Quota, error)
return r0, r1
}
// GetByRef provides a mock function with given fields: ctx, reference, referenceID
func (_m *Controller) GetByRef(ctx context.Context, reference string, referenceID string) (*models.Quota, error) {
ret := _m.Called(ctx, reference, referenceID)
var r0 *models.Quota
if rf, ok := ret.Get(0).(func(context.Context, string, string) *models.Quota); ok {
r0 = rf(ctx, reference, referenceID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*models.Quota)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
r1 = rf(ctx, reference, referenceID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// IsEnabled provides a mock function with given fields: ctx, reference, referenceID
func (_m *Controller) IsEnabled(ctx context.Context, reference string, referenceID string) (bool, error) {
ret := _m.Called(ctx, reference, referenceID)
@ -102,13 +127,20 @@ func (_m *Controller) IsEnabled(ctx context.Context, reference string, reference
return r0, r1
}
// Refresh provides a mock function with given fields: ctx, reference, referenceID
func (_m *Controller) Refresh(ctx context.Context, reference string, referenceID string) error {
ret := _m.Called(ctx, reference, referenceID)
// Refresh provides a mock function with given fields: ctx, reference, referenceID, options
func (_m *Controller) Refresh(ctx context.Context, reference string, referenceID string, options ...quota.Option) error {
_va := make([]interface{}, len(options))
for _i := range options {
_va[_i] = options[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, reference, referenceID)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok {
r0 = rf(ctx, reference, referenceID)
if rf, ok := ret.Get(0).(func(context.Context, string, string, ...quota.Option) error); ok {
r0 = rf(ctx, reference, referenceID, options...)
} else {
r0 = ret.Error(0)
}

27
src/testing/pkg/quota/manager.go
View File

@ -81,8 +81,31 @@ func (_m *Manager) Get(ctx context.Context, id int64) (*models.Quota, error) {
return r0, r1
}
// GetForUpdate provides a mock function with given fields: ctx, reference, referenceID
func (_m *Manager) GetForUpdate(ctx context.Context, reference string, referenceID string) (*models.Quota, error) {
// GetByRef provides a mock function with given fields: ctx, reference, referenceID
func (_m *Manager) GetByRef(ctx context.Context, reference string, referenceID string) (*models.Quota, error) {
ret := _m.Called(ctx, reference, referenceID)
var r0 *models.Quota
if rf, ok := ret.Get(0).(func(context.Context, string, string) *models.Quota); ok {
r0 = rf(ctx, reference, referenceID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*models.Quota)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
r1 = rf(ctx, reference, referenceID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetByRefForUpdate provides a mock function with given fields: ctx, reference, referenceID
func (_m *Manager) GetByRefForUpdate(ctx context.Context, reference string, referenceID string) (*models.Quota, error) {
ret := _m.Called(ctx, reference, referenceID)
var r0 *models.Quota