Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-06-25 06:15:04 +02:00
Code Issues Projects Releases Wiki Activity

feat(quota,webhook): send quota webhook for put and mount blob

Browse Source 
Closes #11712

Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2020-04-28 00:59:16 +00:00
parent fb90bc23f2
commit b1c9d452ce
9 changed files with 182 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/controller/event/handler/webhook/quota/quota.go
View File

@ -17,6 +17,7 @@ package quota
import (
"errors"
"fmt"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/controller/event/handler/util"
@ -101,11 +102,14 @@ func constructQuotaPayload(event *event.QuotaEvent) (*model.Payload, error) {
Custom: quotaCustom,
},
}
resource := &notifyModel.Resource{
Tag: event.Resource.Tag,
Digest: event.Resource.Digest,
if event.Resource != nil {
resource := &notifyModel.Resource{
Tag: event.Resource.Tag,
Digest: event.Resource.Digest,
}
payload.EventData.Resources = append(payload.EventData.Resources, resource)
}
payload.EventData.Resources = append(payload.EventData.Resources, resource)
return payload, nil
}

18
src/controller/event/metadata/quota.go
View File

@ -34,17 +34,21 @@ func (q *QuotaMetaData) Resolve(evt *event.Event) error {
return errors.New("not supported quota status")
}
evt.Topic = topic
evt.Data = &event2.QuotaEvent{
data := &event2.QuotaEvent{
EventType: topic,
Project: q.Project,
Resource: &event2.ImgResource{
OccurAt: q.OccurAt,
RepoName: q.RepoName,
Msg: q.Msg,
}
if q.Tag != "" || q.Digest != "" {
data.Resource = &event2.ImgResource{
Tag: q.Tag,
Digest: q.Digest,
},
OccurAt: q.OccurAt,
RepoName: q.RepoName,
Msg: q.Msg,
}
}
evt.Topic = topic
evt.Data = data
return nil
}

50
src/controller/event/metadata/quota_test.go
View File

@ -15,34 +15,66 @@
package metadata
import (
"testing"
event2 "github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/pkg/notifier/event"
"github.com/stretchr/testify/suite"
"testing"
)
type quotaEventTestSuite struct {
suite.Suite
}
func (r *quotaEventTestSuite) TestResolveOfDeleteRepositoryEventMetadata() {
func (suite *quotaEventTestSuite) TestResolveOfDeleteRepositoryEventMetadata() {
e := &event.Event{}
metadata := &QuotaMetaData{
RepoName: "library/hello-world",
Tag: "latest",
Digest: "sha256:123absd",
Digest: "sha256:469b2a896fbc1123f4894ac8023003f23588967aee5c2cbbce15d6b49dfe048e",
Level: 1,
Msg: "quota exceed",
}
err := metadata.Resolve(e)
r.Require().Nil(err)
r.Equal(event2.TopicQuotaExceed, e.Topic)
r.Require().NotNil(e.Data)
suite.Nil(err)
suite.Equal(event2.TopicQuotaExceed, e.Topic)
suite.NotNil(e.Data)
data, ok := e.Data.(*event2.QuotaEvent)
r.Require().True(ok)
r.Equal("library/hello-world", data.Resource)
suite.True(ok)
suite.Equal("library/hello-world", data.RepoName)
suite.NotNil(data.Resource)
suite.Equal("latest", data.Resource.Tag)
suite.Equal("sha256:469b2a896fbc1123f4894ac8023003f23588967aee5c2cbbce15d6b49dfe048e", data.Resource.Digest)
}
func (suite *quotaEventTestSuite) TestNoResource() {
e := &event.Event{}
metadata := &QuotaMetaData{
RepoName: "library/hello-world",
Level: 2,
Msg: "quota exceed",
}
err := metadata.Resolve(e)
suite.Nil(err)
suite.Equal(event2.TopicQuotaWarning, e.Topic)
suite.NotNil(e.Data)
data, ok := e.Data.(*event2.QuotaEvent)
suite.True(ok)
suite.Equal("library/hello-world", data.RepoName)
suite.Nil(data.Resource)
}
func (suite *quotaEventTestSuite) TestUnsupportedStatus() {
e := &event.Event{}
metadata := &QuotaMetaData{
RepoName: "library/hello-world",
Level: 3,
Msg: "quota exceed",
}
err := metadata.Resolve(e)
suite.Error(err)
}
func TestQuotaEventTestSuite(t *testing.T) {
suite.Run(t, &repositoryEventTestSuite{})
suite.Run(t, &quotaEventTestSuite{})
}

6
src/server/middleware/quota/post_initiate_blob_upload.go
View File

@ -27,8 +27,10 @@ import (
// PostInitiateBlobUploadMiddleware middleware to add blob to project after mount blob success
func PostInitiateBlobUploadMiddleware() func(http.Handler) http.Handler {
return RequestMiddleware(RequestConfig{
ReferenceObject: projectReferenceObject,
Resources: postInitiateBlobUploadResources,
ReferenceObject: projectReferenceObject,
Resources: postInitiateBlobUploadResources,
ResourcesExceeded: projectResourcesEvent(1),
ResourcesWarning: projectResourcesEvent(2),
})
}

2
src/server/middleware/quota/post_initiate_blob_upload_test.go
View File

@ -20,6 +20,7 @@ import (
"testing"
"github.com/goharbor/harbor/src/pkg/blob"
"github.com/goharbor/harbor/src/pkg/quota"
"github.com/goharbor/harbor/src/pkg/types"
"github.com/goharbor/harbor/src/testing/mock"
"github.com/stretchr/testify/suite"
@ -60,6 +61,7 @@ func (suite *PostInitiateBlobUploadMiddlewareTestSuite) TestMiddleware() {
f := args.Get(4).(func() error)
f()
})
mock.OnAnything(suite.quotaController, "GetByRef").Return(&quota.Quota{}, nil).Once()
req := httptest.NewRequest(http.MethodPost, url, nil)
rr := httptest.NewRecorder()

6
src/server/middleware/quota/put_blob_upload.go
View File

@ -27,8 +27,10 @@ import (
// PutBlobUploadMiddleware middleware to request storage resource for the project
func PutBlobUploadMiddleware() func(http.Handler) http.Handler {
return RequestMiddleware(RequestConfig{
ReferenceObject: projectReferenceObject,
Resources: putBlobUploadResources,
ReferenceObject: projectReferenceObject,
Resources: putBlobUploadResources,
ResourcesExceeded: projectResourcesEvent(1),
ResourcesWarning: projectResourcesEvent(2),
})
}

43
src/server/middleware/quota/put_blob_upload_test.go
View File

@ -21,6 +21,10 @@ import (
"strconv"
"testing"
commonmodels "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/quota"
"github.com/goharbor/harbor/src/pkg/types"
"github.com/goharbor/harbor/src/testing/mock"
"github.com/stretchr/testify/suite"
@ -92,6 +96,7 @@ func (suite *PutBlobUploadMiddlewareTestSuite) TestBlobNotExist() {
f := args.Get(4).(func() error)
f()
})
mock.OnAnything(suite.quotaController, "GetByRef").Return(&quota.Quota{}, nil).Once()
req := suite.makeRequest(100)
rr := httptest.NewRecorder()
@ -110,6 +115,44 @@ func (suite *PutBlobUploadMiddlewareTestSuite) TestBlobExistFailed() {
suite.Equal(http.StatusInternalServerError, rr.Code)
}
func (suite *PutBlobUploadMiddlewareTestSuite) TestResourcesExceeded() {
mock.OnAnything(suite.quotaController, "IsEnabled").Return(true, nil)
mock.OnAnything(suite.blobController, "Exist").Return(false, nil)
mock.OnAnything(suite.projectController, "Get").Return(&commonmodels.Project{}, nil)
{
var errs quota.Errors
errs = errs.Add(quota.NewResourceOverflowError(types.ResourceStorage, 100, 100, 110))
mock.OnAnything(suite.quotaController, "Request").Return(errs).Once()
req := suite.makeRequest(100)
eveCtx := notification.NewEventCtx()
req = req.WithContext(notification.NewContext(req.Context(), eveCtx))
rr := httptest.NewRecorder()
suite.handler.ServeHTTP(rr, req)
suite.NotEqual(http.StatusOK, rr.Code)
suite.Equal(1, eveCtx.Events.Len())
}
{
var errs quota.Errors
errs = errs.Add(quota.NewResourceOverflowError(types.ResourceStorage, 100, 100, 110))
err := errors.DeniedError(errs).WithMessage("Quota exceeded when processing the request of %v", errs)
mock.OnAnything(suite.quotaController, "Request").Return(err).Once()
req := suite.makeRequest(100)
eveCtx := notification.NewEventCtx()
req = req.WithContext(notification.NewContext(req.Context(), eveCtx))
rr := httptest.NewRecorder()
suite.handler.ServeHTTP(rr, req)
suite.NotEqual(http.StatusOK, rr.Code)
suite.Equal(1, eveCtx.Events.Len())
}
}
func TestPutBlobUploadMiddlewareTestSuite(t *testing.T) {
suite.Run(t, &PutBlobUploadMiddlewareTestSuite{})
}

63
src/server/middleware/quota/put_manifest.go
View File

@ -15,18 +15,12 @@
package quota
import (
"io/ioutil"
"net/http"
"strconv"
"time"
"github.com/goharbor/harbor/src/controller/blob"
"github.com/goharbor/harbor/src/controller/event/metadata"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/blob/models"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/pkg/notifier/event"
"github.com/goharbor/harbor/src/pkg/types"
)
@ -35,25 +29,11 @@ func PutManifestMiddleware() func(http.Handler) http.Handler {
return RequestMiddleware(RequestConfig{
ReferenceObject: projectReferenceObject,
Resources: putManifestResources,
ResourcesExceeded: putManifestResourcesEvent(1),
ResourcesWarning: putManifestResourcesEvent(2),
ResourcesExceeded: projectResourcesEvent(1),
ResourcesWarning: projectResourcesEvent(2),
})
}
var (
unmarshalManifest = func(r *http.Request) (distribution.Manifest, distribution.Descriptor, error) {
lib.NopCloseRequest(r)
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return nil, distribution.Descriptor{}, err
}
contentType := r.Header.Get("Content-Type")
return distribution.UnmarshalManifest(contentType, body)
}
)
func putManifestResources(r *http.Request, reference, referenceID string) (types.ResourceList, error) {
logger := log.G(r.Context()).WithFields(log.Fields{"middleware": "quota", "action": "request", "url": r.URL.Path})
@ -99,42 +79,3 @@ func putManifestResources(r *http.Request, reference, referenceID string) (types
return types.ResourceList{types.ResourceStorage: size}, nil
}
func putManifestResourcesEvent(level int) func(*http.Request, string, string, string) event.Metadata {
return func(r *http.Request, reference, referenceID string, message string) event.Metadata {
ctx := r.Context()
logger := log.G(ctx).WithFields(log.Fields{"middleware": "quota", "action": "request", "url": r.URL.Path})
_, descriptor, err := unmarshalManifest(r)
if err != nil {
logger.Errorf("unmarshal manifest failed, error: %v", err)
return nil
}
projectID, _ := strconv.ParseInt(referenceID, 10, 64)
project, err := projectController.Get(ctx, projectID)
if err != nil {
logger.Errorf("get project %d failed, error: %v", projectID, err)
return nil
}
path := r.URL.EscapedPath()
var tag string
if ref := distribution.ParseReference(path); !distribution.IsDigest(ref) {
tag = ref
}
return &metadata.QuotaMetaData{
Project: project,
Tag: tag,
Digest: descriptor.Digest.String(),
RepoName: distribution.ParseName(path),
Level: level,
Msg: message,
OccurAt: time.Now(),
}
}
}

67
src/server/middleware/quota/util.go
View File

@ -16,9 +16,17 @@ package quota
import (
"fmt"
"io/ioutil"
"net/http"
"strconv"
"time"
"github.com/goharbor/harbor/src/controller/event/metadata"
"github.com/goharbor/harbor/src/controller/quota"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/pkg/notifier/event"
"github.com/goharbor/harbor/src/server/middleware/util"
)
@ -36,3 +44,62 @@ func projectReferenceObject(r *http.Request) (string, string, error) {
return quota.ProjectReference, quota.ReferenceID(project.ProjectID), nil
}
var (
unmarshalManifest = func(r *http.Request) (distribution.Manifest, distribution.Descriptor, error) {
lib.NopCloseRequest(r)
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return nil, distribution.Descriptor{}, err
}
contentType := r.Header.Get("Content-Type")
return distribution.UnmarshalManifest(contentType, body)
}
)
func projectResourcesEvent(level int) func(*http.Request, string, string, string) event.Metadata {
return func(r *http.Request, reference, referenceID string, message string) event.Metadata {
ctx := r.Context()
logger := log.G(ctx).WithFields(log.Fields{"middleware": "quota", "action": "request", "url": r.URL.Path})
path := r.URL.EscapedPath()
var (
digest string
tag string
)
if distribution.ManifestURLRegexp.MatchString(path) {
_, descriptor, err := unmarshalManifest(r)
if err != nil {
logger.Errorf("unmarshal manifest failed, error: %v", err)
return nil
}
digest = descriptor.Digest.String()
if ref := distribution.ParseReference(path); !distribution.IsDigest(ref) {
tag = ref
}
}
projectID, _ := strconv.ParseInt(referenceID, 10, 64)
project, err := projectController.Get(ctx, projectID)
if err != nil {
logger.Errorf("get project %d failed, error: %v", projectID, err)
return nil
}
return &metadata.QuotaMetaData{
Project: project,
Tag: tag,
Digest: digest,
RepoName: distribution.ParseName(path),
Level: level,
Msg: message,
OccurAt: time.Now(),
}
}
}