From fe39bb6a2a37789906d8a5549f01183b162bfaf1 Mon Sep 17 00:00:00 2001 From: He Weiwei Date: Wed, 18 Mar 2020 20:24:23 +0800 Subject: [PATCH] feat(quota,notification): notification for quota exceeded and warning (#11123) Signed-off-by: He Weiwei --- src/api/event/metadata/quota.go | 27 +++-- src/api/event/topic.go | 5 +- src/api/quota/controller.go | 5 +- src/core/api/notification_policy.go | 24 +++-- src/pkg/distribution/distribution.go | 17 ++- src/pkg/distribution/distribution_test.go | 22 ++++ src/pkg/notification/notification.go | 13 +++ src/pkg/quota/errors.go | 16 +++ src/pkg/quota/models/quota.go | 37 +++++++ src/pkg/quota/models/quota_test.go | 35 ++++++ .../middleware/notification/notification.go | 32 ++---- .../notification/notification_test.go | 37 +++++-- src/server/middleware/quota/copy_artifact.go | 57 +++++++++- .../middleware/quota/copy_artifact_test.go | 87 ++++++++++++++- src/server/middleware/quota/put_manifest.go | 48 ++++++++- .../middleware/quota/put_manifest_test.go | 101 ++++++++++++++++++ src/server/middleware/quota/quota.go | 64 +++++++++++ src/server/middleware/quota/quota_test.go | 18 +++- 18 files changed, 574 insertions(+), 71 deletions(-) create mode 100644 src/pkg/quota/models/quota_test.go diff --git a/src/api/event/metadata/quota.go b/src/api/event/metadata/quota.go index 319106a42..8335b46df 100644 --- a/src/api/event/metadata/quota.go +++ b/src/api/event/metadata/quota.go @@ -1,11 +1,12 @@ package metadata import ( + "time" + event2 "github.com/goharbor/harbor/src/api/event" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/pkg/notifier/event" "github.com/pkg/errors" - "time" ) // QuotaMetaData defines quota related event data @@ -24,18 +25,6 @@ type QuotaMetaData struct { // Resolve quota exceed into common image event func (q *QuotaMetaData) Resolve(evt *event.Event) error { var topic string - data := &event2.QuotaEvent{ - EventType: event2.TopicQuotaExceed, - Project: q.Project, - Resource: &event2.ImgResource{ - Tag: q.Tag, - Digest: q.Digest, - }, - OccurAt: q.OccurAt, - RepoName: q.RepoName, - Msg: q.Msg, - } - switch q.Level { case 1: topic = event2.TopicQuotaExceed @@ -46,6 +35,16 @@ func (q *QuotaMetaData) Resolve(evt *event.Event) error { } evt.Topic = topic - evt.Data = data + evt.Data = &event2.QuotaEvent{ + EventType: topic, + Project: q.Project, + Resource: &event2.ImgResource{ + Tag: q.Tag, + Digest: q.Digest, + }, + OccurAt: q.OccurAt, + RepoName: q.RepoName, + Msg: q.Msg, + } return nil } diff --git a/src/api/event/topic.go b/src/api/event/topic.go index 0aa0f9c6d..e7dc245ff 100644 --- a/src/api/event/topic.go +++ b/src/api/event/topic.go @@ -16,11 +16,12 @@ package event import ( "fmt" + "time" + "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/pkg/artifact" "github.com/goharbor/harbor/src/pkg/audit/model" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" - "time" ) // the event consumers can refer to this file to find all topics and the corresponding event structures @@ -38,7 +39,7 @@ const ( TopicScanningFailed = "SCANNING_FAILED" TopicScanningCompleted = "SCANNING_COMPLETED" // QuotaExceedTopic is topic for quota warning event, the usage reaches the warning bar of limitation, like 85% - TopicQuotaWarning = "QUOTA_WARNNING" + TopicQuotaWarning = "QUOTA_WARNING" TopicQuotaExceed = "QUOTA_EXCEED" TopicUploadChart = "UPLOAD_CHART" TopicDownloadChart = "DOWNLOAD_CHART" diff --git a/src/api/quota/controller.go b/src/api/quota/controller.go index ac3ec3bec..8ab343be6 100644 --- a/src/api/quota/controller.go +++ b/src/api/quota/controller.go @@ -167,9 +167,8 @@ func (c *controller) reserveResources(ctx context.Context, reference, referenceI newReserved := types.Add(reserved, resources) - newUsed := types.Add(used, newReserved) - if err := quota.IsSafe(hardLimits, used, newUsed, false); err != nil { - return ierror.DeniedError(nil).WithMessage("Quota exceeded when processing the request of %v", err) + if err := quota.IsSafe(hardLimits, types.Add(used, reserved), types.Add(used, newReserved), false); err != nil { + return ierror.DeniedError(err).WithMessage("Quota exceeded when processing the request of %v", err) } if err := c.setReservedResources(ctx, reference, referenceID, newReserved); err != nil { diff --git a/src/core/api/notification_policy.go b/src/core/api/notification_policy.go index b1e230771..cae117859 100755 --- a/src/core/api/notification_policy.go +++ b/src/core/api/notification_policy.go @@ -3,16 +3,15 @@ package api import ( "errors" "fmt" - "github.com/goharbor/harbor/src/api/event" "net/http" "strconv" "time" - "github.com/goharbor/harbor/src/common/utils/log" - + "github.com/goharbor/harbor/src/api/event" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/pkg/notification" ) @@ -373,15 +372,24 @@ func getLastTriggerTimeGroupByEventType(eventType string, policyID int64) (time. } func initSupportedEvents() map[string]struct{} { - var supportedEventTypes = make(map[string]struct{}) - eventTypes := []string{event.TopicPushArtifact, event.TopicPullArtifact, - event.TopicDeleteArtifact, event.TopicUploadChart, event.TopicDeleteChart, - event.TopicDownloadChart, event.TopicQuotaExceed, event.TopicScanningFailed, - event.TopicScanningCompleted} + eventTypes := []string{ + event.TopicPushArtifact, + event.TopicPullArtifact, + event.TopicDeleteArtifact, + event.TopicUploadChart, + event.TopicDeleteChart, + event.TopicDownloadChart, + event.TopicQuotaExceed, + event.TopicQuotaWarning, + event.TopicScanningFailed, + event.TopicScanningCompleted, + } + var supportedEventTypes = make(map[string]struct{}) for _, eventType := range eventTypes { supportedEventTypes[eventType] = struct{}{} } + return supportedEventTypes } diff --git a/src/pkg/distribution/distribution.go b/src/pkg/distribution/distribution.go index b98bb87b7..17413b168 100644 --- a/src/pkg/distribution/distribution.go +++ b/src/pkg/distribution/distribution.go @@ -46,7 +46,7 @@ var ( var ( name = fmt.Sprintf("(?P%s)", ref.NameRegexp) - reference = fmt.Sprintf("(?P(%s|%s))", ref.TagRegexp, ref.DigestRegexp) + reference = fmt.Sprintf("(?P((%s)|(%s)))", ref.DigestRegexp, ref.TagRegexp) sessionID = "(?P[a-zA-Z0-9-_.=]+)" // BlobUploadURLRegexp regexp which match blob upload url @@ -74,6 +74,16 @@ func ParseName(path string) string { return "" } +// ParseReference returns digest or tag from distribution API URL path +func ParseReference(path string) string { + m := utils.FindNamedMatches(ManifestURLRegexp, path) + if len(m) > 0 { + return m["reference"] + } + + return "" +} + // ParseProjectName returns project name from distribution API URL path func ParseProjectName(path string) string { projectName, _ := utils.ParseRepository(ParseName(path)) @@ -109,3 +119,8 @@ func ParseRef(s string) (string, string, error) { return repository, reference, nil } + +// IsDigest returns true when reference is digest +func IsDigest(reference string) bool { + return ref.DigestRegexp.MatchString(reference) +} diff --git a/src/pkg/distribution/distribution_test.go b/src/pkg/distribution/distribution_test.go index ee8c332d1..7a1b41e10 100644 --- a/src/pkg/distribution/distribution_test.go +++ b/src/pkg/distribution/distribution_test.go @@ -18,6 +18,7 @@ import ( "testing" _ "github.com/docker/distribution/manifest/manifestlist" + _ "github.com/docker/distribution/manifest/ocischema" _ "github.com/docker/distribution/manifest/schema1" _ "github.com/docker/distribution/manifest/schema2" ) @@ -96,3 +97,24 @@ func TestParseProjectName(t *testing.T) { }) } } + +func TestParseReference(t *testing.T) { + type args struct { + path string + } + tests := []struct { + name string + args args + want string + }{ + {"tag", args{"/v2/library/photon/manifests/2.0"}, "2.0"}, + {"digest", args{"/v2/library/photon/manifests/sha256:c52fca2e807cb7807cfd831d6df45a332d5826a97f886f7da0e9c61842f9ce1e"}, "sha256:c52fca2e807cb7807cfd831d6df45a332d5826a97f886f7da0e9c61842f9ce1e"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ParseReference(tt.args.path); got != tt.want { + t.Errorf("ParseReference() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/pkg/notification/notification.go b/src/pkg/notification/notification.go index 5811ba556..db658ab84 100755 --- a/src/pkg/notification/notification.go +++ b/src/pkg/notification/notification.go @@ -3,6 +3,7 @@ package notification import ( "container/list" "context" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/pkg/notification/hook" "github.com/goharbor/harbor/src/pkg/notification/job" @@ -59,6 +60,14 @@ type EventCtx struct { MustNotify bool } +// NewEventCtx returns instance of EventCtx +func NewEventCtx() *EventCtx { + return &EventCtx{ + Events: list.New(), + MustNotify: false, + } +} + // NewContext returns new context with event func NewContext(ctx context.Context, ec *EventCtx) context.Context { if ctx == nil { @@ -69,6 +78,10 @@ func NewContext(ctx context.Context, ec *EventCtx) context.Context { // AddEvent add events into request context, the event will be sent by the notification middleware eventually. func AddEvent(ctx context.Context, m n_event.Metadata, notify ...bool) { + if m == nil { + return + } + e, ok := ctx.Value(eventKey{}).(*EventCtx) if !ok { log.Debug("request has not event list, cannot add event into context") diff --git a/src/pkg/quota/errors.go b/src/pkg/quota/errors.go index 162cf536f..19f135eaf 100644 --- a/src/pkg/quota/errors.go +++ b/src/pkg/quota/errors.go @@ -63,6 +63,22 @@ func (errs Errors) Error() string { return strings.Join(errors, "; ") } +// Exceeded returns exceeded errors from errs +func (errs Errors) Exceeded() error { + var exceeded Errors + for _, err := range errs.GetErrors() { + if _, ok := err.(*ResourceOverflow); ok { + exceeded = exceeded.Add(err) + } + } + + if len(exceeded) == 0 { + return nil + } + + return exceeded +} + // ResourceOverflow ... type ResourceOverflow struct { Resource types.ResourceName diff --git a/src/pkg/quota/models/quota.go b/src/pkg/quota/models/quota.go index 5b81cfa82..0021f749b 100644 --- a/src/pkg/quota/models/quota.go +++ b/src/pkg/quota/models/quota.go @@ -16,6 +16,7 @@ package models import ( "encoding/json" + "fmt" "time" "github.com/goharbor/harbor/src/pkg/quota/driver" @@ -86,3 +87,39 @@ func (q *Quota) SetUsed(used types.ResourceList) *Quota { return q } + +// GetWarningResources returns resource names which exceeded the warning percent +func (q *Quota) GetWarningResources(warningPercent int) ([]types.ResourceName, error) { + if warningPercent < 0 || warningPercent > 100 { + return nil, fmt.Errorf("bad warningPercent") + } + + hardLimits, err := q.GetHard() + if err != nil { + return nil, err + } + + usage, err := q.GetUsed() + if err != nil { + return nil, err + } + + var resources []types.ResourceName + for resource, used := range usage { + limited, ok := hardLimits[resource] + if !ok { + return nil, fmt.Errorf("resource %s not found in hard limits", resource) + } + + if limited == types.UNLIMITED { + continue + } + + // used / limited >= warningPercent / 100 + if used*100 >= limited*int64(warningPercent) { + resources = append(resources, resource) + } + } + + return resources, nil +} diff --git a/src/pkg/quota/models/quota_test.go b/src/pkg/quota/models/quota_test.go new file mode 100644 index 000000000..adaf12150 --- /dev/null +++ b/src/pkg/quota/models/quota_test.go @@ -0,0 +1,35 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "testing" + + "github.com/goharbor/harbor/src/pkg/types" + "github.com/stretchr/testify/assert" +) + +func TestGetWarningResources(t *testing.T) { + assert := assert.New(t) + + q := Quota{} + + q.SetHard(types.ResourceList{types.ResourceCount: 3}) + q.SetUsed(types.ResourceList{types.ResourceCount: 3}) + + resources, err := q.GetWarningResources(85) + assert.Nil(err) + assert.Len(resources, 1) +} diff --git a/src/server/middleware/notification/notification.go b/src/server/middleware/notification/notification.go index 53d08b9fa..e2ee37b77 100644 --- a/src/server/middleware/notification/notification.go +++ b/src/server/middleware/notification/notification.go @@ -15,38 +15,24 @@ package notification import ( - "container/list" - "github.com/goharbor/harbor/src/pkg/notification" - "github.com/goharbor/harbor/src/server/middleware" "net/http" "github.com/goharbor/harbor/src/internal" - evt "github.com/goharbor/harbor/src/pkg/notifier/event" + "github.com/goharbor/harbor/src/pkg/notification" + "github.com/goharbor/harbor/src/pkg/notifier/event" + "github.com/goharbor/harbor/src/server/middleware" ) -// publishEvent publishes the events in the context, it ensures publish happens after transaction success. -func publishEvent(es *list.List) { - if es == nil { - return - } - for e := es.Front(); e != nil; e = e.Next() { - evt.BuildAndPublish(e.Value.(evt.Metadata)) - } - return -} - // Middleware sends the notification after transaction success func Middleware(skippers ...middleware.Skipper) func(http.Handler) http.Handler { return middleware.New(func(w http.ResponseWriter, r *http.Request, next http.Handler) { res := internal.NewResponseRecorder(w) - eveCtx := ¬ification.EventCtx{ - Events: list.New(), - MustNotify: false, - } - ctx := notification.NewContext(r.Context(), eveCtx) - next.ServeHTTP(res, r.WithContext(ctx)) - if res.Success() || eveCtx.MustNotify { - publishEvent(eveCtx.Events) + evc := notification.NewEventCtx() + next.ServeHTTP(res, r.WithContext(notification.NewContext(r.Context(), evc))) + if res.Success() || evc.MustNotify { + for e := evc.Events.Front(); e != nil; e = e.Next() { + event.BuildAndPublish(e.Value.(event.Metadata)) + } } }, skippers...) } diff --git a/src/server/middleware/notification/notification_test.go b/src/server/middleware/notification/notification_test.go index 5af00a6a0..de23e708a 100644 --- a/src/server/middleware/notification/notification_test.go +++ b/src/server/middleware/notification/notification_test.go @@ -1,28 +1,43 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package notification import ( "context" "fmt" - "github.com/goharbor/harbor/src/api/event/metadata" - pkg_art "github.com/goharbor/harbor/src/pkg/artifact" - "github.com/goharbor/harbor/src/pkg/notification" - "github.com/stretchr/testify/suite" "net/http" "net/http/httptest" "testing" + + "github.com/goharbor/harbor/src/api/event/metadata" + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/notification" + "github.com/stretchr/testify/suite" ) -type NotificatoinMiddlewareTestSuite struct { +type NotificationMiddlewareTestSuite struct { suite.Suite } -func (suite *NotificatoinMiddlewareTestSuite) TestMiddleware() { +func (suite *NotificationMiddlewareTestSuite) TestMiddleware() { next := func() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) notification.AddEvent(r.Context(), &metadata.DeleteArtifactEventMetadata{ Ctx: context.Background(), - Artifact: &pkg_art.Artifact{ + Artifact: &artifact.Artifact{ ProjectID: 1, RepositoryID: 2, RepositoryName: "library/hello-world", @@ -38,13 +53,13 @@ func (suite *NotificatoinMiddlewareTestSuite) TestMiddleware() { suite.Equal(http.StatusAccepted, res.Code) } -func (suite *NotificatoinMiddlewareTestSuite) TestMiddlewareMustNotify() { +func (suite *NotificationMiddlewareTestSuite) TestMiddlewareMustNotify() { next := func() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) notification.AddEvent(r.Context(), &metadata.DeleteArtifactEventMetadata{ Ctx: context.Background(), - Artifact: &pkg_art.Artifact{ + Artifact: &artifact.Artifact{ ProjectID: 1, RepositoryID: 2, RepositoryName: "library/hello-world", @@ -60,6 +75,6 @@ func (suite *NotificatoinMiddlewareTestSuite) TestMiddlewareMustNotify() { suite.Equal(http.StatusInternalServerError, res.Code) } -func TestNotificatoinMiddlewareTestSuite(t *testing.T) { - suite.Run(t, &NotificatoinMiddlewareTestSuite{}) +func TestNotificationMiddlewareTestSuite(t *testing.T) { + suite.Run(t, &NotificationMiddlewareTestSuite{}) } diff --git a/src/server/middleware/quota/copy_artifact.go b/src/server/middleware/quota/copy_artifact.go index 10d0e9672..6335776a7 100644 --- a/src/server/middleware/quota/copy_artifact.go +++ b/src/server/middleware/quota/copy_artifact.go @@ -33,12 +33,15 @@ import ( "path" "strconv" "strings" + "time" "github.com/goharbor/harbor/src/api/artifact" + "github.com/goharbor/harbor/src/api/event/metadata" "github.com/goharbor/harbor/src/common/utils/log" ierror "github.com/goharbor/harbor/src/internal/error" "github.com/goharbor/harbor/src/pkg/blob" "github.com/goharbor/harbor/src/pkg/distribution" + "github.com/goharbor/harbor/src/pkg/notifier/event" "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/types" ) @@ -46,8 +49,10 @@ import ( // CopyArtifactMiddleware middleware to request count and storage resources for copy artifact API func CopyArtifactMiddleware() func(http.Handler) http.Handler { return RequestMiddleware(RequestConfig{ - ReferenceObject: projectReferenceObject, - Resources: copyArtifactResources, + ReferenceObject: projectReferenceObject, + Resources: copyArtifactResources, + ResourcesExceeded: copyArtifactResourcesEvent(1), + ResourcesWarning: copyArtifactResourcesEvent(2), }) } @@ -140,3 +145,51 @@ func copyArtifactResources(r *http.Request, reference, referenceID string) (type return types.ResourceList{types.ResourceCount: copyCount, types.ResourceStorage: size}, nil } + +func copyArtifactResourcesEvent(level int) func(*http.Request, string, string, string) event.Metadata { + return func(r *http.Request, reference, referenceID string, message string) event.Metadata { + ctx := r.Context() + + logger := log.G(ctx).WithFields(log.Fields{"middleware": "quota", "action": "request", "url": r.URL.Path}) + + query := r.URL.Query() + from := query.Get("from") + if from == "" { + // this will never be happened + return nil + } + + repository, reference, err := distribution.ParseRef(from) + if err != nil { + // this will never be happened + return nil + } + + art, err := artifactController.GetByReference(ctx, repository, reference, nil) + if err != nil { + logger.Errorf("get artifact %s failed, error: %v", from, err) + } + + projectID, _ := strconv.ParseInt(referenceID, 10, 64) + project, err := projectController.Get(ctx, projectID) + if err != nil { + logger.Errorf("get artifact %s failed, error: %v", from, err) + return nil + } + + var tag string + if distribution.IsDigest(reference) { + tag = reference + } + + return &metadata.QuotaMetaData{ + Project: project, + Tag: tag, + Digest: art.Digest, + RepoName: parseRepositoryName(r.URL.EscapedPath()), + Level: level, + Msg: message, + OccurAt: time.Now(), + } + } +} diff --git a/src/server/middleware/quota/copy_artifact_test.go b/src/server/middleware/quota/copy_artifact_test.go index 19c9ea067..25ad5bd00 100644 --- a/src/server/middleware/quota/copy_artifact_test.go +++ b/src/server/middleware/quota/copy_artifact_test.go @@ -28,7 +28,19 @@ package quota -import "testing" +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/goharbor/harbor/src/api/artifact" + commonmodels "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/pkg/notification" + "github.com/goharbor/harbor/src/pkg/quota" + "github.com/goharbor/harbor/src/pkg/types" + "github.com/goharbor/harbor/src/testing/mock" + "github.com/stretchr/testify/suite" +) func Test_parseRepositoryName(t *testing.T) { type args struct { @@ -52,3 +64,76 @@ func Test_parseRepositoryName(t *testing.T) { }) } } + +type CopyArtifactMiddlewareTestSuite struct { + RequestMiddlewareTestSuite + + artifact *artifact.Artifact +} + +func (suite *CopyArtifactMiddlewareTestSuite) SetupTest() { + suite.RequestMiddlewareTestSuite.SetupTest() + + mock.OnAnything(suite.quotaController, "IsEnabled").Return(true, nil) + + suite.artifact = &artifact.Artifact{} + + mock.OnAnything(suite.artifactController, "GetByReference").Return(suite.artifact, nil) + mock.OnAnything(suite.artifactController, "Walk").Return(nil).Run(func(args mock.Arguments) { + walkFn := args.Get(2).(func(*artifact.Artifact) error) + walkFn(suite.artifact) + }) + + mock.OnAnything(suite.projectController, "Get").Return(&commonmodels.Project{}, nil) +} + +func (suite *CopyArtifactMiddlewareTestSuite) TestResourcesWarning() { + mock.OnAnything(suite.blobController, "List").Return(nil, nil) + mock.OnAnything(suite.blobController, "FindMissingAssociationsForProject").Return(nil, nil) + mock.OnAnything(suite.quotaController, "Request").Return(nil).Run(func(args mock.Arguments) { + f := args.Get(4).(func() error) + f() + }) + + mock.OnAnything(suite.artifactController, "Count").Return(int64(0), nil) + + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + { + q := "a.Quota{} + q.SetHard(types.ResourceList{types.ResourceCount: 100}) + q.SetUsed(types.ResourceList{types.ResourceCount: 50}) + mock.OnAnything(suite.quotaController, "GetByRef").Return(q, nil).Once() + + req := httptest.NewRequest(http.MethodPut, "/v2/library/photon/manifests/2.0?from=library/photon:2.0.1", nil) + eveCtx := notification.NewEventCtx() + req = req.WithContext(notification.NewContext(req.Context(), eveCtx)) + rr := httptest.NewRecorder() + + CopyArtifactMiddleware()(next).ServeHTTP(rr, req) + suite.Equal(http.StatusOK, rr.Code) + suite.Equal(0, eveCtx.Events.Len()) + } + + { + q := "a.Quota{} + q.SetHard(types.ResourceList{types.ResourceCount: 100}) + q.SetUsed(types.ResourceList{types.ResourceCount: 85}) + mock.OnAnything(suite.quotaController, "GetByRef").Return(q, nil).Once() + + req := httptest.NewRequest(http.MethodPut, "/v2/library/photon/manifests/2.0?from=library/photon:2.0.1", nil) + eveCtx := notification.NewEventCtx() + req = req.WithContext(notification.NewContext(req.Context(), eveCtx)) + rr := httptest.NewRecorder() + + CopyArtifactMiddleware()(next).ServeHTTP(rr, req) + suite.Equal(http.StatusOK, rr.Code) + suite.Equal(1, eveCtx.Events.Len()) + } +} + +func TestCopyArtifactMiddlewareTestSuite(t *testing.T) { + suite.Run(t, &CopyArtifactMiddlewareTestSuite{}) +} diff --git a/src/server/middleware/quota/put_manifest.go b/src/server/middleware/quota/put_manifest.go index a0105b3be..75a894701 100644 --- a/src/server/middleware/quota/put_manifest.go +++ b/src/server/middleware/quota/put_manifest.go @@ -18,20 +18,25 @@ import ( "io/ioutil" "net/http" "strconv" + "time" "github.com/goharbor/harbor/src/api/blob" + "github.com/goharbor/harbor/src/api/event/metadata" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/internal" "github.com/goharbor/harbor/src/pkg/blob/models" "github.com/goharbor/harbor/src/pkg/distribution" + "github.com/goharbor/harbor/src/pkg/notifier/event" "github.com/goharbor/harbor/src/pkg/types" ) // PutManifestMiddleware middleware to request count and storage resources for the project func PutManifestMiddleware() func(http.Handler) http.Handler { return RequestMiddleware(RequestConfig{ - ReferenceObject: projectReferenceObject, - Resources: putManifestResources, + ReferenceObject: projectReferenceObject, + Resources: putManifestResources, + ResourcesExceeded: putManifestResourcesEvent(1), + ResourcesWarning: putManifestResourcesEvent(2), }) } @@ -94,3 +99,42 @@ func putManifestResources(r *http.Request, reference, referenceID string) (types return types.ResourceList{types.ResourceCount: 1, types.ResourceStorage: size}, nil } + +func putManifestResourcesEvent(level int) func(*http.Request, string, string, string) event.Metadata { + return func(r *http.Request, reference, referenceID string, message string) event.Metadata { + ctx := r.Context() + + logger := log.G(ctx).WithFields(log.Fields{"middleware": "quota", "action": "request", "url": r.URL.Path}) + + _, descriptor, err := unmarshalManifest(r) + if err != nil { + logger.Errorf("unmarshal manifest failed, error: %v", err) + return nil + } + + projectID, _ := strconv.ParseInt(referenceID, 10, 64) + project, err := projectController.Get(ctx, projectID) + if err != nil { + logger.Errorf("get project %d failed, error: %v", projectID, err) + + return nil + } + + path := r.URL.EscapedPath() + + var tag string + if ref := distribution.ParseReference(path); !distribution.IsDigest(ref) { + tag = ref + } + + return &metadata.QuotaMetaData{ + Project: project, + Tag: tag, + Digest: descriptor.Digest.String(), + RepoName: distribution.ParseName(path), + Level: level, + Msg: message, + OccurAt: time.Now(), + } + } +} diff --git a/src/server/middleware/quota/put_manifest_test.go b/src/server/middleware/quota/put_manifest_test.go index f8ce45e06..2ca177a17 100644 --- a/src/server/middleware/quota/put_manifest_test.go +++ b/src/server/middleware/quota/put_manifest_test.go @@ -21,8 +21,12 @@ import ( "testing" "github.com/docker/distribution/manifest/schema2" + commonmodels "github.com/goharbor/harbor/src/common/models" + ierror "github.com/goharbor/harbor/src/internal/error" "github.com/goharbor/harbor/src/pkg/blob/models" "github.com/goharbor/harbor/src/pkg/distribution" + "github.com/goharbor/harbor/src/pkg/notification" + "github.com/goharbor/harbor/src/pkg/quota" "github.com/goharbor/harbor/src/pkg/types" "github.com/goharbor/harbor/src/testing/mock" distributiontesting "github.com/goharbor/harbor/src/testing/pkg/distribution" @@ -90,6 +94,7 @@ func (suite *PutManifestMiddlewareTestSuite) TestMiddleware() { f := args.Get(4).(func() error) f() }) + mock.OnAnything(suite.quotaController, "GetByRef").Return("a.Quota{}, nil).Once() req := httptest.NewRequest(http.MethodPut, "/v2/library/photon/manifests/2.0", nil) rr := httptest.NewRecorder() @@ -116,6 +121,7 @@ func (suite *PutManifestMiddlewareTestSuite) TestMiddleware() { f := args.Get(4).(func() error) f() }) + mock.OnAnything(suite.quotaController, "GetByRef").Return("a.Quota{}, nil).Once() req := httptest.NewRequest(http.MethodPut, "/v2/library/photon/manifests/2.0", nil) rr := httptest.NewRecorder() @@ -142,6 +148,7 @@ func (suite *PutManifestMiddlewareTestSuite) TestMiddleware() { f := args.Get(4).(func() error) f() }) + mock.OnAnything(suite.quotaController, "GetByRef").Return("a.Quota{}, nil).Once() req := httptest.NewRequest(http.MethodPut, "/v2/library/photon/manifests/2.0", nil) rr := httptest.NewRecorder() @@ -168,6 +175,7 @@ func (suite *PutManifestMiddlewareTestSuite) TestMiddleware() { f := args.Get(4).(func() error) f() }) + mock.OnAnything(suite.quotaController, "GetByRef").Return("a.Quota{}, nil).Once() req := httptest.NewRequest(http.MethodPut, "/v2/library/photon/manifests/2.0", nil) rr := httptest.NewRecorder() @@ -177,6 +185,99 @@ func (suite *PutManifestMiddlewareTestSuite) TestMiddleware() { } } +func (suite *PutManifestMiddlewareTestSuite) TestResourcesExceeded() { + mock.OnAnything(suite.quotaController, "IsEnabled").Return(true, nil) + mock.OnAnything(suite.blobController, "Exist").Return(false, nil) + mock.OnAnything(suite.blobController, "FindMissingAssociationsForProject").Return(nil, nil) + mock.OnAnything(suite.projectController, "Get").Return(&commonmodels.Project{}, nil) + + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + { + var errs quota.Errors + errs = errs.Add(quota.NewResourceOverflowError(types.ResourceCount, 10, 10, 11)) + errs = errs.Add(quota.NewResourceOverflowError(types.ResourceStorage, 100, 100, 110)) + mock.OnAnything(suite.quotaController, "Request").Return(errs).Once() + + req := httptest.NewRequest(http.MethodPut, "/v2/library/photon/manifests/2.0", nil) + eveCtx := notification.NewEventCtx() + req = req.WithContext(notification.NewContext(req.Context(), eveCtx)) + rr := httptest.NewRecorder() + + PutManifestMiddleware()(next).ServeHTTP(rr, req) + suite.NotEqual(http.StatusOK, rr.Code) + suite.Equal(1, eveCtx.Events.Len()) + } + + { + var errs quota.Errors + errs = errs.Add(quota.NewResourceOverflowError(types.ResourceCount, 10, 10, 11)) + errs = errs.Add(quota.NewResourceOverflowError(types.ResourceStorage, 100, 100, 110)) + + err := ierror.DeniedError(errs).WithMessage("Quota exceeded when processing the request of %v", errs) + mock.OnAnything(suite.quotaController, "Request").Return(err).Once() + + req := httptest.NewRequest(http.MethodPut, "/v2/library/photon/manifests/2.0", nil) + eveCtx := notification.NewEventCtx() + req = req.WithContext(notification.NewContext(req.Context(), eveCtx)) + rr := httptest.NewRecorder() + + PutManifestMiddleware()(next).ServeHTTP(rr, req) + suite.NotEqual(http.StatusOK, rr.Code) + suite.Equal(1, eveCtx.Events.Len()) + } +} + +func (suite *PutManifestMiddlewareTestSuite) TestResourcesWarning() { + mock.OnAnything(suite.quotaController, "IsEnabled").Return(true, nil) + mock.OnAnything(suite.blobController, "Exist").Return(false, nil) + mock.OnAnything(suite.blobController, "FindMissingAssociationsForProject").Return(nil, nil) + + mock.OnAnything(suite.quotaController, "Request").Return(nil).Run(func(args mock.Arguments) { + f := args.Get(4).(func() error) + f() + }) + mock.OnAnything(suite.projectController, "Get").Return(&commonmodels.Project{}, nil) + + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + { + q := "a.Quota{} + q.SetHard(types.ResourceList{types.ResourceCount: 100}) + q.SetUsed(types.ResourceList{types.ResourceCount: 50}) + mock.OnAnything(suite.quotaController, "GetByRef").Return(q, nil).Once() + + req := httptest.NewRequest(http.MethodPut, "/v2/library/photon/manifests/2.0", nil) + eveCtx := notification.NewEventCtx() + req = req.WithContext(notification.NewContext(req.Context(), eveCtx)) + rr := httptest.NewRecorder() + + PutManifestMiddleware()(next).ServeHTTP(rr, req) + suite.Equal(http.StatusOK, rr.Code) + suite.Equal(0, eveCtx.Events.Len()) + } + + { + q := "a.Quota{} + q.SetHard(types.ResourceList{types.ResourceCount: 100}) + q.SetUsed(types.ResourceList{types.ResourceCount: 85}) + mock.OnAnything(suite.quotaController, "GetByRef").Return(q, nil).Once() + + req := httptest.NewRequest(http.MethodPut, "/v2/library/photon/manifests/2.0", nil) + eveCtx := notification.NewEventCtx() + req = req.WithContext(notification.NewContext(req.Context(), eveCtx)) + rr := httptest.NewRecorder() + + PutManifestMiddleware()(next).ServeHTTP(rr, req) + suite.Equal(http.StatusOK, rr.Code) + suite.Equal(1, eveCtx.Events.Len()) + } +} + func TestPutManifestMiddlewareTestSuite(t *testing.T) { suite.Run(t, &PutManifestMiddlewareTestSuite{}) } diff --git a/src/server/middleware/quota/quota.go b/src/server/middleware/quota/quota.go index 069203e9c..ac480eac9 100644 --- a/src/server/middleware/quota/quota.go +++ b/src/server/middleware/quota/quota.go @@ -18,9 +18,13 @@ import ( "errors" "fmt" "net/http" + "strings" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/internal" + "github.com/goharbor/harbor/src/pkg/notification" + "github.com/goharbor/harbor/src/pkg/notifier/event" + "github.com/goharbor/harbor/src/pkg/quota" "github.com/goharbor/harbor/src/pkg/types" serror "github.com/goharbor/harbor/src/server/error" "github.com/goharbor/harbor/src/server/middleware" @@ -37,10 +41,23 @@ type RequestConfig struct { // Resources returns request resources for the reference object Resources func(r *http.Request, reference, referenceID string) (types.ResourceList, error) + + // ResourcesWarningPercent value from 0 to 100 + ResourcesWarningPercent int + + // ResourcesWarning returns event which will be notified when resources usage exceeded the wanring percent + ResourcesWarning func(r *http.Request, reference, referenceID string, message string) event.Metadata + + // ResourcesExceeded returns event which will be notified when resources exceeded the limitation + ResourcesExceeded func(r *http.Request, reference, referenceID string, message string) event.Metadata } // RequestMiddleware middleware which request resources func RequestMiddleware(config RequestConfig, skippers ...middleware.Skipper) func(http.Handler) http.Handler { + if config.ResourcesWarningPercent == 0 { + config.ResourcesWarningPercent = 85 // default 85% + } + return middleware.New(func(w http.ResponseWriter, r *http.Request, next http.Handler) { logger := log.G(r.Context()).WithFields(log.Fields{"middleware": "quota", "action": "request", "url": r.URL.Path}) @@ -101,7 +118,54 @@ func RequestMiddleware(config RequestConfig, skippers ...middleware.Skipper) fun return nil }) + if err == nil && config.ResourcesWarning != nil { + tryWarningNotification := func() { + q, err := quotaController.GetByRef(r.Context(), reference, referenceID) + if err != nil { + logger.Warningf("get quota of %s %s failed, error: %v", reference, referenceID, err) + return + } + + resources, err := q.GetWarningResources(config.ResourcesWarningPercent) + if err != nil { + logger.Warningf("get warning resources failed, error: %v", err) + return + } + + if len(resources) == 0 { + logger.Warningf("not warning resources found") + return + } + + hardLimits, _ := q.GetHard() + used, _ := q.GetUsed() + + var parts []string + for _, resource := range resources { + s := fmt.Sprintf("resource %s used %s of %s", + resource, resource.FormatValue(used[resource]), resource.FormatValue(hardLimits[resource])) + parts = append(parts, s) + } + + message := fmt.Sprintf("quota usage reach %d%%: %s", config.ResourcesWarningPercent, strings.Join(parts, "; ")) + evt := config.ResourcesWarning(r, reference, referenceID, message) + notification.AddEvent(r.Context(), evt, true) + } + + tryWarningNotification() + } + if err != nil && err != errNonSuccess { + if config.ResourcesExceeded != nil { + var errs quota.Errors // NOTE: quota.Errors is slice, so we need var here not pointer + if errors.As(err, &errs) { + if exceeded := errs.Exceeded(); exceeded != nil { + evt := config.ResourcesExceeded(r, reference, referenceID, exceeded.Error()) + notification.AddEvent(r.Context(), evt, true) + } + } + } + res.Reset() serror.SendError(res, err) } diff --git a/src/server/middleware/quota/quota_test.go b/src/server/middleware/quota/quota_test.go index f77f3cbcc..e609f03d2 100644 --- a/src/server/middleware/quota/quota_test.go +++ b/src/server/middleware/quota/quota_test.go @@ -20,11 +20,13 @@ import ( "net/http/httptest" "testing" + "github.com/goharbor/harbor/src/api/artifact" "github.com/goharbor/harbor/src/api/blob" "github.com/goharbor/harbor/src/api/project" "github.com/goharbor/harbor/src/api/quota" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/pkg/types" + artifacttesting "github.com/goharbor/harbor/src/testing/api/artifact" blobtesting "github.com/goharbor/harbor/src/testing/api/blob" projecttesting "github.com/goharbor/harbor/src/testing/api/project" quotatesting "github.com/goharbor/harbor/src/testing/api/quota" @@ -35,8 +37,11 @@ import ( type RequestMiddlewareTestSuite struct { suite.Suite - originallBlobController blob.Controller - blobController *blobtesting.Controller + originalArtifactController artifact.Controller + artifactController *artifacttesting.Controller + + originalBlobController blob.Controller + blobController *blobtesting.Controller originalProjectController project.Controller projectController *projecttesting.Controller @@ -46,7 +51,11 @@ type RequestMiddlewareTestSuite struct { } func (suite *RequestMiddlewareTestSuite) SetupTest() { - suite.originallBlobController = blobController + suite.originalArtifactController = artifactController + suite.artifactController = &artifacttesting.Controller{} + artifactController = suite.artifactController + + suite.originalBlobController = blobController suite.blobController = &blobtesting.Controller{} blobController = suite.blobController @@ -62,7 +71,8 @@ func (suite *RequestMiddlewareTestSuite) SetupTest() { } func (suite *RequestMiddlewareTestSuite) TearDownTest() { - blobController = suite.originallBlobController + artifactController = suite.originalArtifactController + blobController = suite.originalBlobController projectController = suite.originalProjectController quotaController = suite.originallQuotaController }