Merge pull request #11765 from heww/quota-webhook-enhancement

feat(quota,webhook): send quota webhook for put and mount blob

src/controller/event/handler/webhook/quota/quota.go

@@ -17,6 +17,7 @@ package quota
 import (
 	"errors"
 	"fmt"
+
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/controller/event"
 	"github.com/goharbor/harbor/src/controller/event/handler/util"
@@ -101,11 +102,14 @@ func constructQuotaPayload(event *event.QuotaEvent) (*model.Payload, error) {
 			Custom: quotaCustom,
 		},
 	}
-	resource := &notifyModel.Resource{
+
-		Tag:    event.Resource.Tag,
+	if event.Resource != nil {
-		Digest: event.Resource.Digest,
+		resource := &notifyModel.Resource{
+			Tag:    event.Resource.Tag,
+			Digest: event.Resource.Digest,
+		}
+		payload.EventData.Resources = append(payload.EventData.Resources, resource)
 	}
-	payload.EventData.Resources = append(payload.EventData.Resources, resource)
 
 	return payload, nil
 }

src/controller/event/metadata/quota.go

@@ -34,17 +34,21 @@ func (q *QuotaMetaData) Resolve(evt *event.Event) error {
 		return errors.New("not supported quota status")
 	}
 
-	evt.Topic = topic
+	data := &event2.QuotaEvent{
-	evt.Data = &event2.QuotaEvent{
 		EventType: topic,
 		Project:   q.Project,
-		Resource: &event2.ImgResource{
+		OccurAt:   q.OccurAt,
+		RepoName:  q.RepoName,
+		Msg:       q.Msg,
+	}
+	if q.Tag != "" || q.Digest != "" {
+		data.Resource = &event2.ImgResource{
 			Tag:    q.Tag,
 			Digest: q.Digest,
-		},
+		}
-		OccurAt:  q.OccurAt,
-		RepoName: q.RepoName,
-		Msg:      q.Msg,
 	}
+
+	evt.Topic = topic
+	evt.Data = data
 	return nil
 }

src/controller/event/metadata/quota_test.go

@@ -15,34 +15,66 @@
 package metadata
 
 import (
+	"testing"
+
 	event2 "github.com/goharbor/harbor/src/controller/event"
 	"github.com/goharbor/harbor/src/pkg/notifier/event"
 	"github.com/stretchr/testify/suite"
-	"testing"
 )
 
 type quotaEventTestSuite struct {
 	suite.Suite
 }
 
-func (r *quotaEventTestSuite) TestResolveOfDeleteRepositoryEventMetadata() {
+func (suite *quotaEventTestSuite) TestResolveOfDeleteRepositoryEventMetadata() {
 	e := &event.Event{}
 	metadata := &QuotaMetaData{
 		RepoName: "library/hello-world",
 		Tag:      "latest",
-		Digest:   "sha256:123absd",
+		Digest:   "sha256:469b2a896fbc1123f4894ac8023003f23588967aee5c2cbbce15d6b49dfe048e",
 		Level:    1,
 		Msg:      "quota exceed",
 	}
 	err := metadata.Resolve(e)
-	r.Require().Nil(err)
+	suite.Nil(err)
-	r.Equal(event2.TopicQuotaExceed, e.Topic)
+	suite.Equal(event2.TopicQuotaExceed, e.Topic)
-	r.Require().NotNil(e.Data)
+	suite.NotNil(e.Data)
 	data, ok := e.Data.(*event2.QuotaEvent)
-	r.Require().True(ok)
+	suite.True(ok)
-	r.Equal("library/hello-world", data.Resource)
+	suite.Equal("library/hello-world", data.RepoName)
+	suite.NotNil(data.Resource)
+	suite.Equal("latest", data.Resource.Tag)
+	suite.Equal("sha256:469b2a896fbc1123f4894ac8023003f23588967aee5c2cbbce15d6b49dfe048e", data.Resource.Digest)
+}
+
+func (suite *quotaEventTestSuite) TestNoResource() {
+	e := &event.Event{}
+	metadata := &QuotaMetaData{
+		RepoName: "library/hello-world",
+		Level:    2,
+		Msg:      "quota exceed",
+	}
+	err := metadata.Resolve(e)
+	suite.Nil(err)
+	suite.Equal(event2.TopicQuotaWarning, e.Topic)
+	suite.NotNil(e.Data)
+	data, ok := e.Data.(*event2.QuotaEvent)
+	suite.True(ok)
+	suite.Equal("library/hello-world", data.RepoName)
+	suite.Nil(data.Resource)
+}
+
+func (suite *quotaEventTestSuite) TestUnsupportedStatus() {
+	e := &event.Event{}
+	metadata := &QuotaMetaData{
+		RepoName: "library/hello-world",
+		Level:    3,
+		Msg:      "quota exceed",
+	}
+	err := metadata.Resolve(e)
+	suite.Error(err)
 }
 
 func TestQuotaEventTestSuite(t *testing.T) {
-	suite.Run(t, &repositoryEventTestSuite{})
+	suite.Run(t, &quotaEventTestSuite{})
 }

src/server/middleware/quota/post_initiate_blob_upload.go

@@ -27,8 +27,10 @@ import (
 // PostInitiateBlobUploadMiddleware middleware to add blob to project after mount blob success
 func PostInitiateBlobUploadMiddleware() func(http.Handler) http.Handler {
 	return RequestMiddleware(RequestConfig{
-		ReferenceObject: projectReferenceObject,
+		ReferenceObject:   projectReferenceObject,
-		Resources:       postInitiateBlobUploadResources,
+		Resources:         postInitiateBlobUploadResources,
+		ResourcesExceeded: projectResourcesEvent(1),
+		ResourcesWarning:  projectResourcesEvent(2),
 	})
 }
 

src/server/middleware/quota/post_initiate_blob_upload_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/goharbor/harbor/src/pkg/blob"
+	"github.com/goharbor/harbor/src/pkg/quota"
 	"github.com/goharbor/harbor/src/pkg/types"
 	"github.com/goharbor/harbor/src/testing/mock"
 	"github.com/stretchr/testify/suite"
@@ -60,6 +61,7 @@ func (suite *PostInitiateBlobUploadMiddlewareTestSuite) TestMiddleware() {
 			f := args.Get(4).(func() error)
 			f()
 		})
+		mock.OnAnything(suite.quotaController, "GetByRef").Return(&quota.Quota{}, nil).Once()
 
 		req := httptest.NewRequest(http.MethodPost, url, nil)
 		rr := httptest.NewRecorder()

src/server/middleware/quota/put_blob_upload.go

@@ -27,8 +27,10 @@ import (
 // PutBlobUploadMiddleware middleware to request storage resource for the project
 func PutBlobUploadMiddleware() func(http.Handler) http.Handler {
 	return RequestMiddleware(RequestConfig{
-		ReferenceObject: projectReferenceObject,
+		ReferenceObject:   projectReferenceObject,
-		Resources:       putBlobUploadResources,
+		Resources:         putBlobUploadResources,
+		ResourcesExceeded: projectResourcesEvent(1),
+		ResourcesWarning:  projectResourcesEvent(2),
 	})
 }
 

src/server/middleware/quota/put_blob_upload_test.go

@@ -21,6 +21,10 @@ import (
 	"strconv"
 	"testing"
 
+	commonmodels "github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/pkg/notification"
+	"github.com/goharbor/harbor/src/pkg/quota"
 	"github.com/goharbor/harbor/src/pkg/types"
 	"github.com/goharbor/harbor/src/testing/mock"
 	"github.com/stretchr/testify/suite"
@@ -92,6 +96,7 @@ func (suite *PutBlobUploadMiddlewareTestSuite) TestBlobNotExist() {
 		f := args.Get(4).(func() error)
 		f()
 	})
+	mock.OnAnything(suite.quotaController, "GetByRef").Return(&quota.Quota{}, nil).Once()
 
 	req := suite.makeRequest(100)
 	rr := httptest.NewRecorder()
@@ -110,6 +115,44 @@ func (suite *PutBlobUploadMiddlewareTestSuite) TestBlobExistFailed() {
 	suite.Equal(http.StatusInternalServerError, rr.Code)
 }
 
+func (suite *PutBlobUploadMiddlewareTestSuite) TestResourcesExceeded() {
+	mock.OnAnything(suite.quotaController, "IsEnabled").Return(true, nil)
+	mock.OnAnything(suite.blobController, "Exist").Return(false, nil)
+	mock.OnAnything(suite.projectController, "Get").Return(&commonmodels.Project{}, nil)
+
+	{
+		var errs quota.Errors
+		errs = errs.Add(quota.NewResourceOverflowError(types.ResourceStorage, 100, 100, 110))
+		mock.OnAnything(suite.quotaController, "Request").Return(errs).Once()
+
+		req := suite.makeRequest(100)
+		eveCtx := notification.NewEventCtx()
+		req = req.WithContext(notification.NewContext(req.Context(), eveCtx))
+		rr := httptest.NewRecorder()
+
+		suite.handler.ServeHTTP(rr, req)
+		suite.NotEqual(http.StatusOK, rr.Code)
+		suite.Equal(1, eveCtx.Events.Len())
+	}
+
+	{
+		var errs quota.Errors
+		errs = errs.Add(quota.NewResourceOverflowError(types.ResourceStorage, 100, 100, 110))
+
+		err := errors.DeniedError(errs).WithMessage("Quota exceeded when processing the request of %v", errs)
+		mock.OnAnything(suite.quotaController, "Request").Return(err).Once()
+
+		req := suite.makeRequest(100)
+		eveCtx := notification.NewEventCtx()
+		req = req.WithContext(notification.NewContext(req.Context(), eveCtx))
+		rr := httptest.NewRecorder()
+
+		suite.handler.ServeHTTP(rr, req)
+		suite.NotEqual(http.StatusOK, rr.Code)
+		suite.Equal(1, eveCtx.Events.Len())
+	}
+}
+
 func TestPutBlobUploadMiddlewareTestSuite(t *testing.T) {
 	suite.Run(t, &PutBlobUploadMiddlewareTestSuite{})
 }

src/server/middleware/quota/put_manifest.go

@@ -15,18 +15,12 @@
 package quota
 
 import (
-	"io/ioutil"
 	"net/http"
 	"strconv"
-	"time"
 
 	"github.com/goharbor/harbor/src/controller/blob"
-	"github.com/goharbor/harbor/src/controller/event/metadata"
-	"github.com/goharbor/harbor/src/lib"
 	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/pkg/blob/models"
-	"github.com/goharbor/harbor/src/pkg/distribution"
-	"github.com/goharbor/harbor/src/pkg/notifier/event"
 	"github.com/goharbor/harbor/src/pkg/types"
 )
 
@@ -35,25 +29,11 @@ func PutManifestMiddleware() func(http.Handler) http.Handler {
 	return RequestMiddleware(RequestConfig{
 		ReferenceObject:   projectReferenceObject,
 		Resources:         putManifestResources,
-		ResourcesExceeded: putManifestResourcesEvent(1),
+		ResourcesExceeded: projectResourcesEvent(1),
-		ResourcesWarning:  putManifestResourcesEvent(2),
+		ResourcesWarning:  projectResourcesEvent(2),
 	})
 }
 
-var (
-	unmarshalManifest = func(r *http.Request) (distribution.Manifest, distribution.Descriptor, error) {
-		lib.NopCloseRequest(r)
-
-		body, err := ioutil.ReadAll(r.Body)
-		if err != nil {
-			return nil, distribution.Descriptor{}, err
-		}
-
-		contentType := r.Header.Get("Content-Type")
-		return distribution.UnmarshalManifest(contentType, body)
-	}
-)
-
 func putManifestResources(r *http.Request, reference, referenceID string) (types.ResourceList, error) {
 	logger := log.G(r.Context()).WithFields(log.Fields{"middleware": "quota", "action": "request", "url": r.URL.Path})
 
@@ -99,42 +79,3 @@ func putManifestResources(r *http.Request, reference, referenceID string) (types
 
 	return types.ResourceList{types.ResourceStorage: size}, nil
 }
-
-func putManifestResourcesEvent(level int) func(*http.Request, string, string, string) event.Metadata {
-	return func(r *http.Request, reference, referenceID string, message string) event.Metadata {
-		ctx := r.Context()
-
-		logger := log.G(ctx).WithFields(log.Fields{"middleware": "quota", "action": "request", "url": r.URL.Path})
-
-		_, descriptor, err := unmarshalManifest(r)
-		if err != nil {
-			logger.Errorf("unmarshal manifest failed, error: %v", err)
-			return nil
-		}
-
-		projectID, _ := strconv.ParseInt(referenceID, 10, 64)
-		project, err := projectController.Get(ctx, projectID)
-		if err != nil {
-			logger.Errorf("get project %d failed, error: %v", projectID, err)
-
-			return nil
-		}
-
-		path := r.URL.EscapedPath()
-
-		var tag string
-		if ref := distribution.ParseReference(path); !distribution.IsDigest(ref) {
-			tag = ref
-		}
-
-		return &metadata.QuotaMetaData{
-			Project:  project,
-			Tag:      tag,
-			Digest:   descriptor.Digest.String(),
-			RepoName: distribution.ParseName(path),
-			Level:    level,
-			Msg:      message,
-			OccurAt:  time.Now(),
-		}
-	}
-}

src/server/middleware/quota/util.go

@@ -16,9 +16,17 @@ package quota
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/http"
+	"strconv"
+	"time"
 
+	"github.com/goharbor/harbor/src/controller/event/metadata"
 	"github.com/goharbor/harbor/src/controller/quota"
+	"github.com/goharbor/harbor/src/lib"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/pkg/distribution"
+	"github.com/goharbor/harbor/src/pkg/notifier/event"
 	"github.com/goharbor/harbor/src/server/middleware/util"
 )
 
@@ -36,3 +44,62 @@ func projectReferenceObject(r *http.Request) (string, string, error) {
 
 	return quota.ProjectReference, quota.ReferenceID(project.ProjectID), nil
 }
+
+var (
+	unmarshalManifest = func(r *http.Request) (distribution.Manifest, distribution.Descriptor, error) {
+		lib.NopCloseRequest(r)
+
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			return nil, distribution.Descriptor{}, err
+		}
+
+		contentType := r.Header.Get("Content-Type")
+		return distribution.UnmarshalManifest(contentType, body)
+	}
+)
+
+func projectResourcesEvent(level int) func(*http.Request, string, string, string) event.Metadata {
+	return func(r *http.Request, reference, referenceID string, message string) event.Metadata {
+		ctx := r.Context()
+
+		logger := log.G(ctx).WithFields(log.Fields{"middleware": "quota", "action": "request", "url": r.URL.Path})
+
+		path := r.URL.EscapedPath()
+
+		var (
+			digest string
+			tag    string
+		)
+		if distribution.ManifestURLRegexp.MatchString(path) {
+			_, descriptor, err := unmarshalManifest(r)
+			if err != nil {
+				logger.Errorf("unmarshal manifest failed, error: %v", err)
+				return nil
+			}
+
+			digest = descriptor.Digest.String()
+			if ref := distribution.ParseReference(path); !distribution.IsDigest(ref) {
+				tag = ref
+			}
+		}
+
+		projectID, _ := strconv.ParseInt(referenceID, 10, 64)
+		project, err := projectController.Get(ctx, projectID)
+		if err != nil {
+			logger.Errorf("get project %d failed, error: %v", projectID, err)
+
+			return nil
+		}
+
+		return &metadata.QuotaMetaData{
+			Project:  project,
+			Tag:      tag,
+			Digest:   digest,
+			RepoName: distribution.ParseName(path),
+			Level:    level,
+			Msg:      message,
+			OccurAt:  time.Now(),
+		}
+	}
+}